diff --git a/Cargo.lock b/Cargo.lock index 21a644b0b..43b14e6e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -161,37 +161,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" -[[package]] -name = "crossbeam-deque" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" - -[[package]] -name = "either" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" - [[package]] name = "env_filter" version = "0.1.4" @@ -231,6 +200,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "escape8259" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5692dd7b5a1978a5aeb0ce83b7655c58ca8efdcb79d21036ea249da95afec2c6" + [[package]] name = "fixedbitset" version = "0.4.2" @@ -452,6 +427,18 @@ version = "0.2.180" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" +[[package]] +name = "libtest-mimic" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5297962ef19edda4ce33aaa484386e0a5b3d7f2f4e037cbeee00503ef6b29d33" +dependencies = [ + "anstream", + "anstyle", + "clap", + "escape8259", +] + [[package]] name = "linux-raw-sys" version = "0.11.0" @@ -623,26 +610,6 @@ dependencies = [ "rand_core", ] -[[package]] -name = "rayon" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - [[package]] name = "regex" version = "1.12.2" @@ -1395,8 +1362,8 @@ dependencies = [ "clap", "heck", "indexmap", + "libtest-mimic", "log", - "rayon", "regex", "serde", "toml", diff --git a/crates/test/Cargo.toml b/crates/test/Cargo.toml index 160afdf1a..da0686a62 100644 --- a/crates/test/Cargo.toml +++ b/crates/test/Cargo.toml @@ -23,7 +23,6 @@ anyhow = { workspace = true } clap = { workspace = true, features = ['env'] } heck = { workspace = true } log = "0.4.26" -rayon = "1.10.0" regex = "1.11.1" serde = { workspace = true } toml = "0.8.20" @@ -39,3 +38,4 @@ wat = { workspace = true } wit-component = { workspace = true } wit-parser = { workspace = true } wit-bindgen-csharp = { workspace = true } +libtest-mimic = "0.8.1" diff --git a/crates/test/src/c.rs b/crates/test/src/c.rs index a396f61d8..3c42aa624 100644 --- a/crates/test/src/c.rs +++ b/crates/test/src/c.rs @@ -33,7 +33,7 @@ struct LangConfig { ldflags: StringList, } -fn clang(runner: &Runner<'_>) -> PathBuf { +fn clang(runner: &Runner) -> PathBuf { let target = &runner.opts.c.c_target; match &runner.opts.c.wasi_sdk_path { Some(path) => path.join(format!("bin/{target}-clang")), @@ -67,20 +67,20 @@ impl LanguageMethods for C { ] } - fn prepare(&self, runner: &mut Runner<'_>) -> Result<()> { + fn prepare(&self, runner: &mut Runner) -> Result<()> { prepare(runner, clang(runner)) } - fn compile(&self, runner: &Runner<'_>, c: &Compile<'_>) -> Result<()> { + fn compile(&self, runner: &Runner, c: &Compile<'_>) -> Result<()> { compile(runner, c, clang(runner)) } - fn verify(&self, runner: &Runner<'_>, v: &Verify<'_>) -> Result<()> { + fn verify(&self, runner: &Runner, v: &Verify<'_>) -> Result<()> { verify(runner, v, clang(runner)) } } -fn prepare(runner: &mut Runner<'_>, compiler: PathBuf) -> Result<()> { +fn prepare(runner: &mut Runner, compiler: PathBuf) -> Result<()> { let cwd = env::current_dir()?; let dir = cwd.join(&runner.opts.artifacts).join("c"); @@ -99,7 +99,7 @@ fn prepare(runner: &mut Runner<'_>, compiler: PathBuf) -> Result<()> { Ok(()) } -fn compile(runner: &Runner<'_>, compile: &Compile<'_>, compiler: PathBuf) -> Result<()> { +fn compile(runner: &Runner, compile: &Compile<'_>, compiler: PathBuf) -> Result<()> { let config = compile.component.deserialize_lang_config::()?; // Compile the C-based bindings to an object file. @@ -162,14 +162,14 @@ fn compile(runner: &Runner<'_>, compile: &Compile<'_>, compiler: PathBuf) -> Res Ok(()) } -fn produces_component(runner: &Runner<'_>) -> bool { +fn produces_component(runner: &Runner) -> bool { match runner.opts.c.c_target.as_str() { "wasm32-wasip1" => false, _ => true, } } -fn verify(runner: &Runner<'_>, verify: &Verify<'_>, compiler: PathBuf) -> Result<()> { +fn verify(runner: &Runner, verify: &Verify<'_>, compiler: PathBuf) -> Result<()> { let mut cmd = Command::new(compiler); cmd.arg( verify diff --git a/crates/test/src/cpp.rs b/crates/test/src/cpp.rs index e2d1b7b83..b7eda2947 100644 --- a/crates/test/src/cpp.rs +++ b/crates/test/src/cpp.rs @@ -19,7 +19,7 @@ struct LangConfig { cflags: StringList, } -fn clangpp(runner: &Runner<'_>) -> PathBuf { +fn clangpp(runner: &Runner) -> PathBuf { match &runner.opts.c.wasi_sdk_path { Some(path) => path.join("bin/wasm32-wasip2-clang++"), None => "wasm32-wasip2-clang++".into(), @@ -57,7 +57,7 @@ impl LanguageMethods for Cpp { } } - fn prepare(&self, runner: &mut crate::Runner<'_>) -> anyhow::Result<()> { + fn prepare(&self, runner: &mut Runner) -> anyhow::Result<()> { let compiler = clangpp(runner); let cwd = std::env::current_dir()?; let dir = cwd.join(&runner.opts.artifacts).join("cpp"); @@ -79,7 +79,7 @@ impl LanguageMethods for Cpp { fn generate_bindings_prepare( &self, - _runner: &Runner<'_>, + _runner: &Runner, bindgen: &crate::Bindgen, dir: &std::path::Path, ) -> anyhow::Result<()> { @@ -106,7 +106,7 @@ impl LanguageMethods for Cpp { Ok(()) } - fn compile(&self, runner: &crate::Runner<'_>, compile: &crate::Compile) -> anyhow::Result<()> { + fn compile(&self, runner: &Runner, compile: &crate::Compile) -> anyhow::Result<()> { let compiler = clangpp(runner); let config = compile.component.deserialize_lang_config::()?; @@ -180,7 +180,7 @@ impl LanguageMethods for Cpp { Ok(()) } - fn verify(&self, runner: &crate::Runner<'_>, verify: &crate::Verify) -> anyhow::Result<()> { + fn verify(&self, runner: &Runner, verify: &crate::Verify) -> anyhow::Result<()> { // for expected let cwd = std::env::current_dir()?; let mut helper_dir2 = cwd; diff --git a/crates/test/src/csharp.rs b/crates/test/src/csharp.rs index f4fb650fb..5b4a50ffd 100644 --- a/crates/test/src/csharp.rs +++ b/crates/test/src/csharp.rs @@ -51,13 +51,13 @@ impl LanguageMethods for Csharp { ) } - fn prepare(&self, runner: &mut Runner<'_>) -> Result<()> { + fn prepare(&self, runner: &mut Runner) -> Result<()> { runner.run_command(dotnet().arg("--version"))?; Ok(()) } - fn compile(&self, runner: &Runner<'_>, compile: &Compile<'_>) -> Result<()> { + fn compile(&self, runner: &Runner, compile: &Compile<'_>) -> Result<()> { let world_name = &compile.component.bindgen.world; let path = &compile.component.path; let test_dir = &compile.bindings_dir; @@ -113,7 +113,7 @@ impl LanguageMethods for Csharp { Ok(()) } - fn verify(&self, runner: &Runner<'_>, verify: &Verify<'_>) -> Result<()> { + fn verify(&self, runner: &Runner, verify: &Verify<'_>) -> Result<()> { let dir = verify.bindings_dir; let name = verify.world; let mut project = wit_bindgen_csharp::CSProject::new(dir.to_path_buf(), &name, "the_world"); diff --git a/crates/test/src/custom.rs b/crates/test/src/custom.rs index 7abc9e09b..402fe65f4 100644 --- a/crates/test/src/custom.rs +++ b/crates/test/src/custom.rs @@ -73,7 +73,7 @@ pub struct Language { } impl Language { - pub fn lookup(runner: &Runner<'_>, language: &str) -> Result { + pub fn lookup(runner: &Runner, language: &str) -> Result { for (ext, script) in runner.opts.custom.custom.iter() { if ext == language { return Ok(Language { @@ -108,7 +108,7 @@ impl LanguageMethods for Language { false } - fn generate_bindings(&self, runner: &Runner<'_>, bindgen: &Bindgen, dir: &Path) -> Result<()> { + fn generate_bindings(&self, runner: &Runner, bindgen: &Bindgen, dir: &Path) -> Result<()> { runner.run_command( Command::new(&self.script) .arg("bindgen") @@ -117,7 +117,7 @@ impl LanguageMethods for Language { ) } - fn prepare(&self, runner: &mut Runner<'_>) -> Result<()> { + fn prepare(&self, runner: &mut Runner) -> Result<()> { let dir = env::current_dir()? .join(&runner.opts.artifacts) .join(&self.extension); @@ -128,7 +128,7 @@ impl LanguageMethods for Language { ) } - fn compile(&self, runner: &Runner<'_>, compile: &Compile<'_>) -> Result<()> { + fn compile(&self, runner: &Runner, compile: &Compile<'_>) -> Result<()> { let dir = env::current_dir()? .join(&runner.opts.artifacts) .join(&self.extension); @@ -144,7 +144,7 @@ impl LanguageMethods for Language { ) } - fn verify(&self, runner: &Runner<'_>, verify: &Verify<'_>) -> Result<()> { + fn verify(&self, runner: &Runner, verify: &Verify<'_>) -> Result<()> { runner.run_command( Command::new(&self.script) .arg("verify") diff --git a/crates/test/src/go.rs b/crates/test/src/go.rs index 23b2a7e4b..cf16d3343 100644 --- a/crates/test/src/go.rs +++ b/crates/test/src/go.rs @@ -34,7 +34,7 @@ impl LanguageMethods for Go { &["--generate-stubs"] } - fn prepare(&self, runner: &mut Runner<'_>) -> Result<()> { + fn prepare(&self, runner: &mut Runner) -> Result<()> { let cwd = env::current_dir()?; let dir = cwd.join(&runner.opts.artifacts).join("go"); let bindings_dir = cwd.join("wit_component"); @@ -56,7 +56,7 @@ impl LanguageMethods for Go { ) } - fn compile(&self, runner: &Runner<'_>, compile: &Compile<'_>) -> Result<()> { + fn compile(&self, runner: &Runner, compile: &Compile<'_>) -> Result<()> { let output = compile.output.with_extension("core.wasm"); // Tests which involve importing and/or exporting more than one @@ -96,7 +96,7 @@ impl LanguageMethods for Go { Ok(()) } - fn verify(&self, runner: &Runner<'_>, verify: &Verify<'_>) -> Result<()> { + fn verify(&self, runner: &Runner, verify: &Verify<'_>) -> Result<()> { replace_bindings_go_mod(runner, verify.bindings_dir)?; runner.run_command( @@ -152,7 +152,7 @@ fn all_paths(path: &Path) -> Result> { Ok(paths) } -fn replace_bindings_go_mod(runner: &Runner<'_>, bindings_dir: &Path) -> Result<()> { +fn replace_bindings_go_mod(runner: &Runner, bindings_dir: &Path) -> Result<()> { let test_crate = PathBuf::from(env!("CARGO_MANIFEST_DIR")); let wit_bindgen_root = test_crate.parent().unwrap().parent().unwrap(); let go_package_path = wit_bindgen_root.join("crates/go/src/package"); diff --git a/crates/test/src/lib.rs b/crates/test/src/lib.rs index 69790972a..e64f13d68 100644 --- a/crates/test/src/lib.rs +++ b/crates/test/src/lib.rs @@ -1,14 +1,14 @@ -use anyhow::{Context, Result, anyhow, bail}; +use anyhow::{Context, Result, bail}; use clap::Parser; -use rayon::prelude::*; +use libtest_mimic::Trial; use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use std::fmt; use std::fs; -use std::io::Write; +use std::mem; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use wasm_encoder::{Encode, Section}; use wit_component::{ComponentEncoder, StringEncoding}; @@ -105,14 +105,39 @@ pub struct Opts { /// Passing `--lang rust` will only test Rust for example. #[clap(short, long, required = true, value_delimiter = ',')] languages: Vec, + + /// Less output per test + #[clap(short, long, conflicts_with = "format")] + quiet: bool, + + /// Number of threads used for parallel testing. + #[clap(long, value_name = "N")] + test_threads: Option, + + /// Only run tests with this exact name. + #[clap(long)] + exact: bool, + + /// A list of filters. Tests whose names contain parts of any of these + /// filters are skipped. + #[clap(long, value_name = "FILTER")] + skip: Vec, + + /// Specifies whether or not to color the output. + #[clap(long, value_name = "auto|always|never")] + color: Option, + + /// Specifies the format of the output. + #[clap(long, value_name = "pretty|terse")] + format: Option, } impl Opts { pub fn run(&self, wit_bindgen: &Path) -> Result<()> { Runner { - opts: self, + opts: self.clone(), rust_state: None, - wit_bindgen, + wit_bindgen: wit_bindgen.to_path_buf(), test_runner: runner::TestRunner::new(&self.runner)?, } .run() @@ -120,6 +145,7 @@ impl Opts { } /// Helper structure representing a discovered `test.wit` file. +#[derive(Clone)] struct Test { /// The name of this test, unique amongst all tests. /// @@ -135,12 +161,14 @@ struct Test { kind: TestKind, } +#[derive(Clone)] enum TestKind { Runtime(Vec), Codegen(PathBuf), } /// Helper structure representing a single component found in a test directory. +#[derive(Clone)] struct Component { /// The name of this component, inferred from the file stem. /// @@ -220,16 +248,16 @@ struct Verify<'a> { } /// Helper structure to package up runtime state associated with executing tests. -struct Runner<'a> { - opts: &'a Opts, +struct Runner { + opts: Opts, rust_state: Option, - wit_bindgen: &'a Path, + wit_bindgen: PathBuf, test_runner: runner::TestRunner, } -impl Runner<'_> { +impl Runner { /// Executes all tests. - fn run(&mut self) -> Result<()> { + fn run(mut self) -> Result<()> { // First step, discover all tests in the specified test directory. let mut tests = HashMap::new(); for test in self.opts.test.iter() { @@ -244,10 +272,9 @@ impl Runner<'_> { } self.prepare_languages(&tests)?; - self.run_codegen_tests(&tests)?; - self.run_runtime_tests(&tests)?; - - println!("PASSED"); + let me = Arc::new(self); + me.run_codegen_tests(&tests)?; + me.run_runtime_tests(&tests)?; Ok(()) } @@ -487,7 +514,7 @@ impl Runner<'_> { } /// Executes all tests that are `TestKind::Codegen`. - fn run_codegen_tests(&mut self, tests: &HashMap) -> Result<()> { + fn run_codegen_tests(self: &Arc, tests: &HashMap) -> Result<()> { let mut codegen_tests = Vec::new(); let languages = self.all_languages(); for (name, config, test) in tests.iter().filter_map(|(name, t)| match &t.kind { @@ -513,7 +540,7 @@ impl Runner<'_> { codegen_tests.push(( language.clone(), - test, + test.to_owned(), name.to_string(), args.clone(), config.clone(), @@ -526,7 +553,7 @@ impl Runner<'_> { } codegen_tests.push(( language.clone(), - test, + test.clone(), format!("{name}-{args_kind}"), args, config.clone(), @@ -539,36 +566,51 @@ impl Runner<'_> { return Ok(()); } - println!("Running {} codegen tests:", codegen_tests.len()); - - let results = codegen_tests - .par_iter() - .map(|(language, test, args_kind, args, config)| { - let should_fail = language.obj().should_fail_verify(args_kind, config, args); - let result = self - .codegen_test(language, test, &args_kind, args, config) - .with_context(|| { - format!("failed to codegen test for `{language}` over {test:?}") - }); - self.update_status(&result, should_fail); - (result, should_fail, language, test, args_kind) - }) - .collect::>(); - - println!(""); - - self.render_errors(results.into_iter().map( - |(result, should_fail, language, test, args_kind)| { - StepResult::new(test.to_str().unwrap(), result) - .should_fail(should_fail) - .metadata("language", language) - .metadata("variant", args_kind) - }, - )); + println!("=== Running codegen tests ==="); + self.run_tests( + codegen_tests + .into_iter() + .map(|(language, test, args_kind, args, config)| { + let me = self.clone(); + let should_fail = language + .obj() + .should_fail_verify(&args_kind, &config, &args); + + let name = format!("{language} {args_kind} {test:?}"); + Trial::test(&name, move || { + let result = me + .codegen_test(&language, &test, &args_kind, &args, &config) + .with_context(|| { + format!("failed to codegen test for `{language}` over {test:?}") + }); + + me.render_error( + StepResult::new(result) + .should_fail(should_fail) + .metadata("language", language) + .metadata("variant", args_kind), + ) + }) + }) + .collect::>(), + ); Ok(()) } + fn run_tests(&self, trials: Vec) { + let args = libtest_mimic::Arguments { + skip: self.opts.skip.clone(), + quiet: self.opts.quiet, + format: self.opts.format, + color: self.opts.color, + test_threads: self.opts.test_threads, + exact: self.opts.exact, + ..Default::default() + }; + libtest_mimic::run(&args, trials).exit_if_failed(); + } + /// Runs a single codegen test. /// /// This will generate bindings for `test` in the `language` specified. The @@ -630,7 +672,7 @@ impl Runner<'_> { } /// Execute all `TestKind::Runtime` tests - fn run_runtime_tests(&mut self, tests: &HashMap) -> Result<()> { + fn run_runtime_tests(self: &Arc, tests: &HashMap) -> Result<()> { let components = tests .values() .filter(|t| match &self.opts.filter { @@ -647,36 +689,36 @@ impl Runner<'_> { .filter(|(_test, component)| self.include_language(&component.language)) .collect::>(); - println!("Compiling {} components:", components.len()); - - // In parallel compile all sources to their binary component - // form. - let compile_results = components - .par_iter() - .map(|(test, component)| { - let path = self - .compile_component(test, component) - .with_context(|| format!("failed to compile component {:?}", component.path)); - self.update_status(&path, false); - (test, component, path) - }) - .collect::>(); - println!(""); - - let mut compilations = Vec::new(); - self.render_errors( - compile_results + println!("=== Compiling components ==="); + let compilations = Arc::new(Mutex::new(Vec::new())); + self.run_tests( + components .into_iter() - .map(|(test, component, result)| match result { - Ok(path) => { - compilations.push((test, component, path)); - StepResult::new("", Ok(())) - } - Err(e) => StepResult::new(&test.name, Err(e)) - .metadata("component", &component.name) - .metadata("path", component.path.display()), - }), + .map(|(test, component)| { + let me = self.clone(); + let compilations = compilations.clone(); + let test = test.clone(); + let component = component.clone(); + Trial::test(&component.path.display().to_string(), move || { + let result = me.compile_component(&test, &component).with_context(|| { + format!("failed to compile component {:?}", component.path) + }); + match result { + Ok(path) => { + compilations.lock().unwrap().push((test, component, path)); + Ok(()) + } + Err(e) => me.render_error( + StepResult::new(Err(e)) + .metadata("component", &component.name) + .metadata("path", component.path.display()), + ), + } + }) + }) + .collect(), ); + let compilations = mem::take(&mut *compilations.lock().unwrap()); // Next, massage the data a bit. Create a map of all tests to where // their components are located. Then perform a product of runners/tests @@ -684,8 +726,8 @@ impl Runner<'_> { // cases. let mut compiled_components = HashMap::new(); for (test, component, path) in compilations { - let list = compiled_components.entry(&test.name).or_insert(Vec::new()); - list.push((*component, path)); + let list = compiled_components.entry(test.name).or_insert(Vec::new()); + list.push((component, path)); } let mut to_run = Vec::new(); @@ -696,51 +738,52 @@ impl Runner<'_> { } } - println!("Running {} runtime tests:", to_run.len()); - - let results = to_run - .par_iter() - .map(|(case_name, (runner, runner_path), test_components)| { - let case = &tests[*case_name]; - let result = self - .runtime_test(case, runner, runner_path, test_components) - .with_context(|| format!("failed to run `{}`", case.name)); - self.update_status(&result, false); - (result, case_name, runner, runner_path, test_components) - }) - .collect::>(); + println!("=== Running runtime tests ==="); - println!(""); - - self.render_errors(results.into_iter().map( - |(result, case_name, runner, runner_path, test_components)| { - let mut result = StepResult::new(case_name, result) - .metadata("runner", runner.path.display()) - .metadata("compiled runner", runner_path.display()); - for (test, path) in test_components { - result = result - .metadata("test", test.path.display()) - .metadata("compiled test", path.display()); - } - result - }, - )); + self.run_tests( + to_run + .into_iter() + .map(|(case_name, (runner, runner_path), test_components)| { + let me = self.clone(); + let mut name = format!("{case_name}"); + for component in [&runner] + .into_iter() + .chain(test_components.iter().map(|p| &p.0)) + { + name.push_str(&format!( + " | {}", + component.path.file_name().unwrap().to_str().unwrap() + )); + } + let case_name = case_name.to_string(); + let runner = runner.clone(); + let runner_path = runner_path.to_path_buf(); + let case = tests[case_name.as_str()].clone(); + Trial::test(&name, move || { + let result = me + .runtime_test(&case, &runner, &runner_path, &test_components) + .with_context(|| format!("failed to run `{}`", case.name)); + me.render_error( + StepResult::new(result) + .metadata("runner", runner.path.display()) + .metadata("compiled runner", runner_path.display()), + ) + }) + }) + .collect(), + ); Ok(()) } /// For the `test` provided, and the selected `runner`, determines all /// permutations of tests from `components` and pushes them on to `to_run`. - fn push_tests<'a>( + fn push_tests( &self, - test: &'a Test, - components: &'a [(&'a Component, PathBuf)], - runner: &'a (&'a Component, PathBuf), - to_run: &mut Vec<( - &'a str, - (&'a Component, &'a Path), - Vec<(&'a Component, &'a Path)>, - )>, + test: &Test, + components: &[(Component, PathBuf)], + runner: &(Component, PathBuf), + to_run: &mut Vec<(String, (Component, PathBuf), Vec<(Component, PathBuf)>)>, ) -> Result<()> { /// Recursive function which walks over `worlds`, the list of worlds /// that `test` expects, one by one. For each world it finds a matching @@ -749,11 +792,11 @@ impl Runner<'_> { /// /// Once `worlds` is empty the `test` list, a temporary vector, is /// cloned and pushed into `commit`. - fn push<'a>( + fn push( worlds: &[String], - components: &'a [(&'a Component, PathBuf)], - test: &mut Vec<(&'a Component, &'a Path)>, - commit: &mut dyn FnMut(Vec<(&'a Component, &'a Path)>), + components: &[(Component, PathBuf)], + test: &mut Vec<(Component, PathBuf)>, + commit: &mut dyn FnMut(Vec<(Component, PathBuf)>), ) -> Result<()> { match worlds.split_first() { Some((world, rest)) => { @@ -761,7 +804,7 @@ impl Runner<'_> { for (component, path) in components { if component.bindgen.world == *world { any = true; - test.push((component, path)); + test.push((component.clone(), path.clone())); push(rest, components, test, commit)?; test.pop(); } @@ -782,7 +825,11 @@ impl Runner<'_> { components, &mut Vec::new(), &mut |test_components| { - to_run.push((&test.name, (runner.0, &runner.1), test_components)); + to_run.push(( + test.name.clone(), + (runner.0.clone(), runner.1.clone()), + test_components, + )); }, ) } @@ -838,7 +885,7 @@ impl Runner<'_> { case: &Test, runner: &Component, runner_wasm: &Path, - test_components: &[(&Component, &Path)], + test_components: &[(Component, PathBuf)], ) -> Result<()> { // If possible use `wasm-compose` to compose the test together. This is // only possible when customization isn't used though. This is also only @@ -871,7 +918,7 @@ impl Runner<'_> { fn compose_wasm_with_wasm_compose( &self, runner_wasm: &Path, - test_components: &[(&Component, &Path)], + test_components: &[(Component, PathBuf)], ) -> Result> { assert!(test_components.len() > 0); let mut last_bytes = None; @@ -906,7 +953,7 @@ impl Runner<'_> { case: &Test, runner: &Component, runner_wasm: &Path, - test_components: &[(&Component, &Path)], + test_components: &[(Component, PathBuf)], ) -> Result> { let document = match &case.config.wac { Some(path) => { @@ -1053,16 +1100,6 @@ status: {}", Ok(()) } - /// "poor man's test output progress" - fn update_status(&self, result: &Result, should_fail: bool) { - if result.is_ok() == !should_fail { - print!("."); - } else { - print!("F"); - } - let _ = std::io::stdout().flush(); - } - /// Returns whether `languages` is included in this testing session. fn include_language(&self, language: &Language) -> bool { self.opts @@ -1071,27 +1108,22 @@ status: {}", .any(|l| l == language.obj().display()) } - fn render_errors<'a>(&self, results: impl Iterator>) { - let mut failures = 0; - for result in results { - let err = match (result.result, result.should_fail) { - (Ok(()), false) | (Err(_), true) => continue, - (Err(e), false) => e, - (Ok(()), true) => anyhow!("test should have failed, but passed"), - }; - failures += 1; - - println!("------ Failure: {} --------", result.name); - for (k, v) in result.metadata { - println!(" {k}: {v}"); - } - println!(" error: {}", format!("{err:?}").replace("\n", "\n ")); - } + fn render_error(&self, result: StepResult<'_>) -> Result<(), libtest_mimic::Failed> { + let err = match (result.result, result.should_fail) { + (Ok(()), false) | (Err(_), true) => return Ok(()), + (Err(e), false) => e, + (Ok(()), true) => return Err("test should have failed, but passed".into()), + }; - if failures > 0 { - println!("{failures} tests FAILED"); - std::process::exit(1); + let mut s = String::new(); + for (k, v) in result.metadata { + s.push_str(&format!(" {k}: {v}\n")); } + s.push_str(&format!( + " error: {}", + format!("{err:?}").replace("\n", "\n ") + )); + Err(s.into()) } } @@ -1110,14 +1142,12 @@ fn has_component_type_sections(wasm: &[u8]) -> bool { struct StepResult<'a> { result: Result<()>, should_fail: bool, - name: &'a str, metadata: Vec<(&'a str, String)>, } impl<'a> StepResult<'a> { - fn new(name: &'a str, result: Result<()>) -> StepResult<'a> { + fn new(result: Result<()>) -> StepResult<'a> { StepResult { - name, result, should_fail: false, metadata: Vec::new(), @@ -1162,12 +1192,12 @@ trait LanguageMethods { /// Performs any one-time preparation necessary for this language, such as /// downloading or caching dependencies. - fn prepare(&self, runner: &mut Runner<'_>) -> Result<()>; + fn prepare(&self, runner: &mut Runner) -> Result<()>; /// Add some files to the generated directory _before_ calling bindgen fn generate_bindings_prepare( &self, - _runner: &Runner<'_>, + _runner: &Runner, _bindgen: &Bindgen, _dir: &Path, ) -> Result<()> { @@ -1177,13 +1207,13 @@ trait LanguageMethods { /// Generates bindings for `component` into `dir`. /// /// Runs `wit-bindgen` in aa subprocess to catch failures such as panics. - fn generate_bindings(&self, runner: &Runner<'_>, bindgen: &Bindgen, dir: &Path) -> Result<()> { + fn generate_bindings(&self, runner: &Runner, bindgen: &Bindgen, dir: &Path) -> Result<()> { let name = match self.bindgen_name() { Some(name) => name, None => return Ok(()), }; self.generate_bindings_prepare(runner, bindgen, dir)?; - let mut cmd = Command::new(runner.wit_bindgen); + let mut cmd = Command::new(&runner.wit_bindgen); cmd.arg(name) .arg(&bindgen.wit_path) .arg("--world") @@ -1232,7 +1262,7 @@ trait LanguageMethods { } /// Performs compilation as specified by `compile`. - fn compile(&self, runner: &Runner<'_>, compile: &Compile) -> Result<()>; + fn compile(&self, runner: &Runner, compile: &Compile) -> Result<()>; /// Returns whether this language is supposed to fail this codegen tests /// given the `config` and `args` for the test. @@ -1240,7 +1270,7 @@ trait LanguageMethods { /// Performs a "check" or a verify that the generated bindings described by /// `Verify` are indeed valid. - fn verify(&self, runner: &Runner<'_>, verify: &Verify) -> Result<()>; + fn verify(&self, runner: &Runner, verify: &Verify) -> Result<()>; } impl Language { diff --git a/crates/test/src/moonbit.rs b/crates/test/src/moonbit.rs index 433f191dd..42c6bbad2 100644 --- a/crates/test/src/moonbit.rs +++ b/crates/test/src/moonbit.rs @@ -1,4 +1,4 @@ -use crate::LanguageMethods; +use crate::{LanguageMethods, Runner}; use anyhow::bail; use serde::Deserialize; use std::process::Command; @@ -28,7 +28,7 @@ impl LanguageMethods for MoonBit { &["--derive-show", "--derive-eq", "--derive-error"] } - fn prepare(&self, runner: &mut crate::Runner<'_>) -> anyhow::Result<()> { + fn prepare(&self, runner: &mut Runner) -> anyhow::Result<()> { println!("Testing if MoonBit toolchain exists..."); if runner .run_command(Command::new("moon").arg("version")) @@ -39,7 +39,7 @@ impl LanguageMethods for MoonBit { Ok(()) } - fn compile(&self, runner: &crate::Runner<'_>, compile: &crate::Compile) -> anyhow::Result<()> { + fn compile(&self, runner: &Runner, compile: &crate::Compile) -> anyhow::Result<()> { let config = compile.component.deserialize_lang_config::()?; // Copy the file to the bindings directory if !config.path.is_empty() { @@ -100,7 +100,7 @@ impl LanguageMethods for MoonBit { config.async_ } - fn verify(&self, runner: &crate::Runner<'_>, verify: &crate::Verify) -> anyhow::Result<()> { + fn verify(&self, runner: &Runner, verify: &crate::Verify) -> anyhow::Result<()> { let mut cmd = Command::new("moon"); cmd.arg("check") .arg("--warn-list") diff --git a/crates/test/src/rust.rs b/crates/test/src/rust.rs index 34c9f2c09..1e9be1b44 100644 --- a/crates/test/src/rust.rs +++ b/crates/test/src/rust.rs @@ -98,7 +98,7 @@ impl LanguageMethods for Rust { &["--stubs"] } - fn prepare(&self, runner: &mut Runner<'_>) -> Result<()> { + fn prepare(&self, runner: &mut Runner) -> Result<()> { let cwd = env::current_dir()?; let opts = &runner.opts.rust; let dir = cwd.join(&runner.opts.artifacts).join("rust"); @@ -164,7 +164,7 @@ path = 'lib.rs' Ok(()) } - fn compile(&self, runner: &Runner<'_>, compile: &Compile) -> Result<()> { + fn compile(&self, runner: &Runner, compile: &Compile) -> Result<()> { let config = compile.component.deserialize_lang_config::()?; // If this rust target doesn't natively produce a component then place @@ -223,7 +223,7 @@ path = 'lib.rs' Ok(()) } - fn verify(&self, runner: &Runner<'_>, verify: &Verify<'_>) -> Result<()> { + fn verify(&self, runner: &Runner, verify: &Verify<'_>) -> Result<()> { let bindings = verify .bindings_dir .join(format!("{}.rs", verify.world.to_snake_case())); @@ -274,7 +274,7 @@ enum Edition { E2024, } -impl Runner<'_> { +impl Runner { fn rustc(&self, edition: Edition) -> Command { let state = self.rust_state.as_ref().unwrap(); let opts = &self.opts.rust; diff --git a/crates/test/src/wat.rs b/crates/test/src/wat.rs index 6209000c2..1cd59cdce 100644 --- a/crates/test/src/wat.rs +++ b/crates/test/src/wat.rs @@ -25,11 +25,11 @@ impl LanguageMethods for Wat { Some(";;@") } - fn prepare(&self, _runner: &mut Runner<'_>) -> Result<()> { + fn prepare(&self, _runner: &mut Runner) -> Result<()> { Ok(()) } - fn compile(&self, runner: &Runner<'_>, compile: &Compile<'_>) -> Result<()> { + fn compile(&self, runner: &Runner, compile: &Compile<'_>) -> Result<()> { let wasm = wat::parse_file(&compile.component.path)?; if wasmparser::Parser::is_component(&wasm) { super::write_if_different(&compile.output, wasm)?; @@ -42,7 +42,7 @@ impl LanguageMethods for Wat { Ok(()) } - fn verify(&self, _runner: &Runner<'_>, _verify: &Verify<'_>) -> Result<()> { + fn verify(&self, _runner: &Runner, _verify: &Verify<'_>) -> Result<()> { // doesn't participate in codegen tests Ok(()) }