Allow running the same tests with different configs in parallel

Author: oli-obk

examples/rustc_twice_with_different_flags.rs

@@ -0,0 +1,29 @@
+#[cfg(feature = "rustc")]
+use std::sync::atomic::Ordering;
+#[cfg(feature = "rustc")]
+use ui_test::{
+    default_file_filter, default_per_file_config, run_tests_generic, status_emitter, Config,
+};
+
+#[cfg(feature = "rustc")]
+#[cfg_attr(test, test)]
+fn main() -> ui_test::color_eyre::Result<()> {
+    let config = Config::rustc("examples_tests/rustc_basic");
+    let abort_check = config.abort_check.clone();
+    ctrlc::set_handler(move || abort_check.store(true, Ordering::Relaxed))?;
+
+    // Compile all `.rs` files in the given directory (relative to your
+    // Cargo.toml) and compare their output against the corresponding
+    // `.stderr` files.
+    run_tests_generic(
+        vec![config.clone(), config],
+        default_file_filter,
+        default_per_file_config,
+        status_emitter::Text::verbose(),
+    )
+}
+
+#[cfg(not(feature = "rustc"))]
+fn main() -> ui_test::color_eyre::Result<()> {
+    Ok(())
+}

src/config.rs

@@ -40,7 +40,7 @@ pub struct Config {
     /// The recommended command to bless failing tests.
     pub bless_command: Option<String>,
     /// Where to dump files like the binaries compiled from tests.
-    /// Defaults to `target/ui` in the current directory.
+    /// Defaults to `target/ui/index_of_config` in the current directory.
     pub out_dir: PathBuf,
     /// Skip test files whose names contain any of these entries.
     pub skip_files: Vec<String>,

src/lib.rs

@@ -182,8 +182,9 @@ pub fn run_tests_generic(
         return Ok(());
     }
 
-    for config in &mut configs {
+    for (i, config) in configs.iter_mut().enumerate() {
         config.fill_host_and_target()?;
+        config.out_dir.push(i.to_string())
     }
 
     let mut results = vec![];

tests/integration.rs

@@ -73,7 +73,10 @@ fn main() -> Result<()> {
     config.filter(" +[0-9]+: .*\n", "");
     config.filter("                +at \\.?/.*\n", "");
     config.filter(" running on .*", "");
-    config.stdout_filter("/target/[^/]+/[^/]+/debug", "/target/$$TMP/$$TRIPLE/debug");
+    config.stdout_filter(
+        "/target/[^/]+/[0-9]+/[^/]+/debug",
+        "/target/$$TMP/$$TRIPLE/debug",
+    );
     config.stdout_filter("/target/.tmp[^/ \"]+", "/target/$$TMP");
     // Normalize proc macro filenames on windows to their linux repr
     config.stdout_filter("/([^/\\.]+)\\.dll", "/lib$1.so");

tests/integrations/basic-bin/tests/ui_tests.rs

@@ -19,7 +19,7 @@ fn main() -> ui_test::color_eyre::Result<()> {
     config.stdout_filter("in ([0-9]m )?[0-9\\.]+s", "");
     config.stderr_filter(r"[^ ]*/\.?cargo/registry/.*/", "$$CARGO_REGISTRY");
     config.stderr_filter(r"\.exe", "");
-    config.stderr_filter("/target/[^/]+/[^/]+/debug", "/target/$$TMP/$$TRIPLE/debug");
+    config.stderr_filter("/target/[^/]+/[0-9]+/[^/]+/debug", "/target/$$TMP/$$TRIPLE/debug");
     config.path_stderr_filter(&std::path::Path::new(path), "$DIR");
 
     // hide binaries generated for successfully passing tests

tests/integrations/basic-fail/Cargo.stderr

@@ -5,15 +5,15 @@ Location:
 error: test failed, to rerun pass `--test ui_tests`
 
 Caused by:
-  process didn't exit successfully: `$DIR/target/ui/tests/integrations/basic-fail/debug/deps/ui_tests-HASH` (exit status: 1)
+  process didn't exit successfully: `$DIR/target/ui/1/tests/integrations/basic-fail/debug/deps/ui_tests-HASH` (exit status: 1)
 Error: tests failed
 
 Location:
     $DIR/src/lib.rs:LL:CC
 error: test failed, to rerun pass `--test ui_tests_diff_only`
 
 Caused by:
-  process didn't exit successfully: `$DIR/target/ui/tests/integrations/basic-fail/debug/deps/ui_tests_diff_only-HASH` (exit status: 1)
+  process didn't exit successfully: `$DIR/target/ui/1/tests/integrations/basic-fail/debug/deps/ui_tests_diff_only-HASH` (exit status: 1)
 Error: failed to parse rustc version info: invalid_foobarlaksdfalsdfj
 
 Caused by:
@@ -23,15 +23,15 @@ Location:
 error: test failed, to rerun pass `--test ui_tests_invalid_program`
 
 Caused by:
-  process didn't exit successfully: `$DIR/target/ui/tests/integrations/basic-fail/debug/deps/ui_tests_invalid_program-HASH` (exit status: 1)
+  process didn't exit successfully: `$DIR/target/ui/1/tests/integrations/basic-fail/debug/deps/ui_tests_invalid_program-HASH` (exit status: 1)
 Error: tests failed
 
 Location:
     $DIR/src/lib.rs:LL:CC
 error: test failed, to rerun pass `--test ui_tests_invalid_program2`
 
 Caused by:
-  process didn't exit successfully: `$DIR/target/ui/tests/integrations/basic-fail/debug/deps/ui_tests_invalid_program2-HASH` (exit status: 1)
+  process didn't exit successfully: `$DIR/target/ui/1/tests/integrations/basic-fail/debug/deps/ui_tests_invalid_program2-HASH` (exit status: 1)
 error: 4 targets failed:
     `--test ui_tests`
     `--test ui_tests_diff_only`

tests/integrations/basic-fail/Cargo.stdout

@@ -558,7 +558,7 @@ full stdout:
 
 
 FAILED TEST: tests/actual_tests_bless/aux_proc_macro_no_main.rs
-command: "rustc" "--error-format=json" "--crate-type=lib" "--out-dir" "$TMP "tests/actual_tests_bless/aux_proc_macro_no_main.rs" "--extern" "the_proc_macro=$DIR/tests/integrations/basic-fail/../../../target/$TMP/tests/actual_tests_bless/auxiliary/libthe_proc_macro.so" "-L" "$DIR/tests/integrations/basic-fail/../../../target/$TMP/tests/actual_tests_bless/auxiliary" "--extern" "basic_fail=$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug/libbasic_fail.rlib" "--extern" "basic_fail=$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug/libbasic_fail-$HASH.rmeta" "-L" "$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug" "-L" "$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug" "--edition" "2021"
+command: "rustc" "--error-format=json" "--crate-type=lib" "--out-dir" "$TMP "tests/actual_tests_bless/aux_proc_macro_no_main.rs" "--extern" "the_proc_macro=$DIR/tests/integrations/basic-fail/../../../target/$TMP/0/tests/actual_tests_bless/auxiliary/libthe_proc_macro.so" "-L" "$DIR/tests/integrations/basic-fail/../../../target/$TMP/0/tests/actual_tests_bless/auxiliary" "--extern" "basic_fail=$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug/libbasic_fail.rlib" "--extern" "basic_fail=$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug/libbasic_fail-$HASH.rmeta" "-L" "$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug" "-L" "$DIR/tests/integrations/basic-fail/../../../target/$TMP/$TRIPLE/debug" "--edition" "2021"
 
 error: there were 1 unmatched diagnostics
  --> tests/actual_tests_bless/aux_proc_macro_no_main.rs:7:8

tests/integrations/dep-fail/Cargo.stderr

@@ -5,6 +5,6 @@ Location:
 error: test failed, to rerun pass `--test ui`
 
 Caused by:
-  process didn't exit successfully: `$DIR/target/ui/tests/integrations/dep-fail/debug/deps/ui-HASH` (exit status: 1)
+  process didn't exit successfully: `$DIR/target/ui/1/tests/integrations/dep-fail/debug/deps/ui-HASH` (exit status: 1)
 error: 1 target failed:
     `--test ui`

tests/integrations/dep-fail/Cargo.stdout

@@ -7,7 +7,7 @@ Building dependencies ... FAILED
 tests/ui/basic_test.rs ... FAILED
 
 FAILED TEST: tests/ui/basic_test.rs
-command: "$CMD" "build" "--color=never" "--quiet" "--jobs" "1" "--target-dir" "$DIR/tests/integrations/dep-fail/target/ui" "--manifest-path" "tested_crate/Cargo.toml"  "--message-format=json"
+command: "$CMD" "build" "--color=never" "--quiet" "--jobs" "1" "--target-dir" "$DIR/tests/integrations/dep-fail/target/ui/0" "--manifest-path" "tested_crate/Cargo.toml"  "--message-format=json"
 
 full stderr:
 error: could not compile `tested_crate` (lib) due to 2 previous errors