chore: improve testsuite

Signed-off-by: Henry Gressmann <[email protected]>

Author: explodingcamera

.cargo/config.toml

@@ -2,4 +2,5 @@
 version-dev="workspaces version --no-git-commit --force tinywasm*"
 test-wasm-1="test --package tinywasm --test test-wasm-1 --release"
 test-wasm-2="test --package tinywasm --test test-wasm-2 --release"
+test-wasm-3="test --package tinywasm --test test-wasm-3 --release"
 test-wast="test --package tinywasm --test test-wast"

Cargo.lock

@@ -20,7 +20,7 @@ tinywasm-types={version="0.9.0-alpha.0", path="../types", default-features=false
 libm={version="0.2", default-features=false}
 
 [dev-dependencies]
-wasm-testsuite={version="0.3.2"}
+wasm-testsuite={version="0.3.3"}
 indexmap="2.7"
 wast={workspace=true}
 eyre={workspace=true}
@@ -45,6 +45,10 @@ harness=false
 name="test-wasm-2"
 harness=false
 
+[[test]]
+name="test-wasm-3"
+harness=false
+
 [[test]]
 name="test-wasm-multi-memory"
 harness=false

crates/tinywasm/Cargo.toml

@@ -0,0 +1 @@
+0.9.0-alpha.0,32305,2591,[{"name":"address.wast","passed":260,"failed":0},{"name":"address64.wast","passed":0,"failed":242},{"name":"align.wast","passed":161,"failed":0},{"name":"align64.wast","passed":83,"failed":73},{"name":"annotations.wast","passed":74,"failed":0},{"name":"binary-leb128.wast","passed":92,"failed":1},{"name":"binary.wast","passed":126,"failed":0},{"name":"block.wast","passed":223,"failed":0},{"name":"br.wast","passed":97,"failed":0},{"name":"br_if.wast","passed":119,"failed":0},{"name":"br_on_non_null.wast","passed":1,"failed":9},{"name":"br_on_null.wast","passed":1,"failed":9},{"name":"br_table.wast","passed":24,"failed":162},{"name":"bulk.wast","passed":117,"failed":0},{"name":"call.wast","passed":91,"failed":0},{"name":"call_indirect.wast","passed":49,"failed":124},{"name":"call_ref.wast","passed":4,"failed":31},{"name":"comments.wast","passed":8,"failed":0},{"name":"const.wast","passed":778,"failed":0},{"name":"conversions.wast","passed":619,"failed":0},{"name":"custom.wast","passed":11,"failed":0},{"name":"data.wast","passed":59,"failed":6},{"name":"elem.wast","passed":137,"failed":14},{"name":"endianness.wast","passed":69,"failed":0},{"name":"endianness64.wast","passed":0,"failed":69},{"name":"exports.wast","passed":97,"failed":0},{"name":"f32.wast","passed":2514,"failed":0},{"name":"f32_bitwise.wast","passed":364,"failed":0},{"name":"f32_cmp.wast","passed":2407,"failed":0},{"name":"f64.wast","passed":2514,"failed":0},{"name":"f64_bitwise.wast","passed":364,"failed":0},{"name":"f64_cmp.wast","passed":2407,"failed":0},{"name":"fac.wast","passed":8,"failed":0},{"name":"float_exprs.wast","passed":927,"failed":0},{"name":"float_literals.wast","passed":179,"failed":0},{"name":"float_memory.wast","passed":90,"failed":0},{"name":"float_memory64.wast","passed":0,"failed":90},{"name":"float_misc.wast","passed":471,"failed":0},{"name":"forward.wast","passed":5,"failed":0},{"name":"func.wast","passed":175,"failed":0},{"name":"func_ptrs.wast","passed":36,"failed":0},{"name":"global.wast","passed":53,"failed":71},{"name":"i32.wast","passed":460,"failed":0},{"name":"i64.wast","passed":416,"failed":0},{"name":"id.wast","passed":7,"failed":0},{"name":"if.wast","passed":241,"failed":0},{"name":"imports.wast","passed":169,"failed":90},{"name":"inline-module.wast","passed":1,"failed":0},{"name":"instance.wast","passed":0,"failed":23},{"name":"int_exprs.wast","passed":108,"failed":0},{"name":"int_literals.wast","passed":51,"failed":0},{"name":"labels.wast","passed":29,"failed":0},{"name":"left-to-right.wast","passed":96,"failed":0},{"name":"linking.wast","passed":142,"failed":21},{"name":"load.wast","passed":118,"failed":0},{"name":"load64.wast","passed":59,"failed":38},{"name":"local_get.wast","passed":36,"failed":0},{"name":"local_init.wast","passed":10,"failed":0},{"name":"local_set.wast","passed":53,"failed":0},{"name":"local_tee.wast","passed":98,"failed":0},{"name":"loop.wast","passed":120,"failed":0},{"name":"memory-multi.wast","passed":6,"failed":0},{"name":"memory.wast","passed":89,"failed":0},{"name":"memory64.wast","passed":14,"failed":53},{"name":"memory_copy.wast","passed":8385,"failed":515},{"name":"memory_fill.wast","passed":164,"failed":36},{"name":"memory_grow.wast","passed":157,"failed":0},{"name":"memory_grow64.wast","passed":0,"failed":49},{"name":"memory_init.wast","passed":307,"failed":173},{"name":"memory_redundancy.wast","passed":8,"failed":0},{"name":"memory_redundancy64.wast","passed":0,"failed":8},{"name":"memory_size.wast","passed":49,"failed":0},{"name":"memory_trap.wast","passed":182,"failed":0},{"name":"memory_trap64.wast","passed":0,"failed":172},{"name":"names.wast","passed":486,"failed":0},{"name":"nop.wast","passed":88,"failed":0},{"name":"obsolete-keywords.wast","passed":11,"failed":0},{"name":"ref.wast","passed":12,"failed":1},{"name":"ref_as_non_null.wast","passed":1,"failed":6},{"name":"ref_func.wast","passed":17,"failed":0},{"name":"ref_is_null.wast","passed":2,"failed":20},{"name":"ref_null.wast","passed":0,"failed":34},{"name":"return.wast","passed":84,"failed":0},{"name":"return_call.wast","passed":11,"failed":34},{"name":"return_call_indirect.wast","passed":26,"failed":50},{"name":"return_call_ref.wast","passed":11,"failed":40},{"name":"select.wast","passed":155,"failed":2},{"name":"skip-stack-guard-page.wast","passed":11,"failed":0},{"name":"stack.wast","passed":7,"failed":0},{"name":"start.wast","passed":20,"failed":0},{"name":"store.wast","passed":111,"failed":0},{"name":"switch.wast","passed":28,"failed":0},{"name":"table-sub.wast","passed":2,"failed":1},{"name":"table.wast","passed":47,"failed":9},{"name":"table_copy.wast","passed":1742,"failed":30},{"name":"table_copy_mixed.wast","passed":4,"failed":0},{"name":"table_fill.wast","passed":45,"failed":35},{"name":"table_get.wast","passed":16,"failed":1},{"name":"table_grow.wast","passed":58,"failed":21},{"name":"table_init.wast","passed":780,"failed":96},{"name":"table_set.wast","passed":26,"failed":2},{"name":"table_size.wast","passed":39,"failed":1},{"name":"tag.wast","passed":1,"failed":8},{"name":"throw.wast","passed":3,"failed":10},{"name":"throw_ref.wast","passed":2,"failed":13},{"name":"token.wast","passed":61,"failed":0},{"name":"traps.wast","passed":36,"failed":0},{"name":"try_table.wast","passed":11,"failed":51},{"name":"type-canon.wast","passed":0,"failed":2},{"name":"type-equivalence.wast","passed":12,"failed":20},{"name":"type-rec.wast","passed":6,"failed":14},{"name":"type.wast","passed":3,"failed":0},{"name":"unreachable.wast","passed":64,"failed":0},{"name":"unreached-invalid.wast","passed":121,"failed":0},{"name":"unreached-valid.wast","passed":2,"failed":11},{"name":"unwind.wast","passed":50,"failed":0},{"name":"utf8-custom-section-id.wast","passed":176,"failed":0},{"name":"utf8-import-field.wast","passed":176,"failed":0},{"name":"utf8-import-module.wast","passed":176,"failed":0},{"name":"utf8-invalid-encoding.wast","passed":176,"failed":0}]

crates/tinywasm/tests/generated/wasm-1.csv

@@ -0,0 +1 @@
+0.9.0-alpha.0,28010,0,[{"name":"address.wast","passed":260,"failed":0},{"name":"align.wast","passed":162,"failed":0},{"name":"binary-leb128.wast","passed":91,"failed":0},{"name":"binary.wast","passed":128,"failed":0},{"name":"block.wast","passed":223,"failed":0},{"name":"br.wast","passed":97,"failed":0},{"name":"br_if.wast","passed":118,"failed":0},{"name":"br_table.wast","passed":174,"failed":0},{"name":"bulk.wast","passed":117,"failed":0},{"name":"call.wast","passed":91,"failed":0},{"name":"call_indirect.wast","passed":172,"failed":0},{"name":"comments.wast","passed":8,"failed":0},{"name":"const.wast","passed":778,"failed":0},{"name":"conversions.wast","passed":619,"failed":0},{"name":"custom.wast","passed":11,"failed":0},{"name":"data.wast","passed":61,"failed":0},{"name":"elem.wast","passed":98,"failed":0},{"name":"endianness.wast","passed":69,"failed":0},{"name":"exports.wast","passed":96,"failed":0},{"name":"f32.wast","passed":2514,"failed":0},{"name":"f32_bitwise.wast","passed":364,"failed":0},{"name":"f32_cmp.wast","passed":2407,"failed":0},{"name":"f64.wast","passed":2514,"failed":0},{"name":"f64_bitwise.wast","passed":364,"failed":0},{"name":"f64_cmp.wast","passed":2407,"failed":0},{"name":"fac.wast","passed":8,"failed":0},{"name":"float_exprs.wast","passed":927,"failed":0},{"name":"float_literals.wast","passed":179,"failed":0},{"name":"float_memory.wast","passed":90,"failed":0},{"name":"float_misc.wast","passed":471,"failed":0},{"name":"forward.wast","passed":5,"failed":0},{"name":"func.wast","passed":172,"failed":0},{"name":"func_ptrs.wast","passed":36,"failed":0},{"name":"global.wast","passed":110,"failed":0},{"name":"i32.wast","passed":460,"failed":0},{"name":"i64.wast","passed":416,"failed":0},{"name":"if.wast","passed":241,"failed":0},{"name":"imports.wast","passed":178,"failed":0},{"name":"inline-module.wast","passed":1,"failed":0},{"name":"int_exprs.wast","passed":108,"failed":0},{"name":"int_literals.wast","passed":51,"failed":0},{"name":"labels.wast","passed":29,"failed":0},{"name":"left-to-right.wast","passed":96,"failed":0},{"name":"linking.wast","passed":132,"failed":0},{"name":"load.wast","passed":97,"failed":0},{"name":"local_get.wast","passed":36,"failed":0},{"name":"local_set.wast","passed":53,"failed":0},{"name":"local_tee.wast","passed":97,"failed":0},{"name":"loop.wast","passed":120,"failed":0},{"name":"memory.wast","passed":88,"failed":0},{"name":"memory_copy.wast","passed":4450,"failed":0},{"name":"memory_fill.wast","passed":100,"failed":0},{"name":"memory_grow.wast","passed":104,"failed":0},{"name":"memory_init.wast","passed":240,"failed":0},{"name":"memory_redundancy.wast","passed":8,"failed":0},{"name":"memory_size.wast","passed":42,"failed":0},{"name":"memory_trap.wast","passed":182,"failed":0},{"name":"names.wast","passed":486,"failed":0},{"name":"nop.wast","passed":88,"failed":0},{"name":"obsolete-keywords.wast","passed":11,"failed":0},{"name":"ref_func.wast","passed":17,"failed":0},{"name":"ref_is_null.wast","passed":16,"failed":0},{"name":"ref_null.wast","passed":3,"failed":0},{"name":"return.wast","passed":84,"failed":0},{"name":"select.wast","passed":148,"failed":0},{"name":"skip-stack-guard-page.wast","passed":11,"failed":0},{"name":"stack.wast","passed":7,"failed":0},{"name":"start.wast","passed":20,"failed":0},{"name":"store.wast","passed":68,"failed":0},{"name":"switch.wast","passed":28,"failed":0},{"name":"table-sub.wast","passed":2,"failed":0},{"name":"table.wast","passed":19,"failed":0},{"name":"table_copy.wast","passed":1728,"failed":0},{"name":"table_fill.wast","passed":45,"failed":0},{"name":"table_get.wast","passed":16,"failed":0},{"name":"table_grow.wast","passed":58,"failed":0},{"name":"table_init.wast","passed":780,"failed":0},{"name":"table_set.wast","passed":26,"failed":0},{"name":"table_size.wast","passed":39,"failed":0},{"name":"token.wast","passed":58,"failed":0},{"name":"traps.wast","passed":36,"failed":0},{"name":"type.wast","passed":3,"failed":0},{"name":"unreachable.wast","passed":64,"failed":0},{"name":"unreached-invalid.wast","passed":118,"failed":0},{"name":"unreached-valid.wast","passed":7,"failed":0},{"name":"unwind.wast","passed":50,"failed":0},{"name":"utf8-custom-section-id.wast","passed":176,"failed":0},{"name":"utf8-import-field.wast","passed":176,"failed":0},{"name":"utf8-import-module.wast","passed":176,"failed":0},{"name":"utf8-invalid-encoding.wast","passed":176,"failed":0}]

crates/tinywasm/tests/generated/wasm-3.csv

@@ -1,13 +1,13 @@
 mod testsuite;
 use eyre::Result;
+use testsuite::TestSuite;
+use wasm_testsuite::data::{spec, SpecVersion};
 
 fn main() -> Result<()> {
-    println!("Skipping 1.0 tests (False positives due to relaxed validation in later versions)");
-    Ok(())
-    // TestSuite::set_log_level(log::LevelFilter::Off);
+    TestSuite::set_log_level(log::LevelFilter::Off);
 
-    // let mut test_suite = TestSuite::new();
-    // test_suite.run_files(spec(&SpecVersion::V1))?;
-    // test_suite.save_csv("./tests/generated/wasm-1.csv", env!("CARGO_PKG_VERSION"))?;
-    // test_suite.report_status()
+    let mut test_suite = TestSuite::new();
+    test_suite.run_files(spec(&SpecVersion::V1))?;
+    test_suite.save_csv("./tests/generated/wasm-1.csv", env!("CARGO_PKG_VERSION"))?;
+    test_suite.report_status()
 }

crates/tinywasm/tests/generated/wasm-latest.csv

@@ -0,0 +1,18 @@
+mod testsuite;
+use eyre::Result;
+use testsuite::TestSuite;
+use wasm_testsuite::data::{spec, SpecVersion};
+
+fn main() -> Result<()> {
+    if std::env::args().find(|x| x == "--enable").is_none() {
+        println!("Skipping wasm-3 tests, use --enable to run");
+        return Ok(());
+    }
+
+    TestSuite::set_log_level(log::LevelFilter::Off);
+
+    let mut test_suite = TestSuite::new();
+    test_suite.run_files(spec(&SpecVersion::V3))?;
+    test_suite.save_csv("./tests/generated/wasm-3.csv", env!("CARGO_PKG_VERSION"))?;
+    test_suite.report_status()
+}

crates/tinywasm/tests/test-wasm-1.rs

@@ -12,37 +12,25 @@ fn main() -> Result<()> {
         bail!("usage: cargo test-wast <wast-file>")
     };
 
-    // cwd for relative paths, absolute paths are kept as-is
-    let cwd = std::env::current_dir()?;
-
-    // if current dir is crates/tinywasm, then we want to go up 2 levels
-    let mut wast_file = if cwd.ends_with("crates/tinywasm") { PathBuf::from("../../") } else { PathBuf::from("./") };
+    let mut cwd = std::env::current_dir()?;
+    if cwd.ends_with("crates/tinywasm/") {
+        cwd.pop();
+        cwd.pop();
+    }
 
-    wast_file.push(&args[1]);
-    let wast_file = cwd.join(wast_file);
+    // if its a folder, run all the wast files in the folder
+    let arg = PathBuf::from(cwd.clone()).join(&args[1]);
+    println!("running tests in {:?}", arg);
 
-    test_wast(wast_file.to_str().expect("wast_file is not a valid path"))?;
-    Ok(())
-}
+    let files = if arg.is_dir() {
+        std::fs::read_dir(&arg)?.map(|entry| entry.map(|e| e.path())).collect::<Result<Vec<_>, _>>()?
+    } else {
+        vec![arg]
+    };
 
-fn test_wast(wast_file: &str) -> Result<()> {
     TestSuite::set_log_level(log::LevelFilter::Debug);
-
-    let args = std::env::args().collect::<Vec<_>>();
-    println!("args: {args:?}");
-
     let mut test_suite = TestSuite::new();
-    println!("running wast file: {wast_file}");
-
-    test_suite.run_paths(&[wast_file])?;
-
-    if test_suite.failed() {
-        println!();
-        test_suite.print_errors();
-        println!();
-        Err(eyre!(format!("{}:\n{:#?}", "failed one or more tests".red().bold(), test_suite,)))
-    } else {
-        println!("\n\npassed all tests:\n{test_suite:#?}");
-        Ok(())
-    }
+    test_suite.run_paths(&files)?;
+    test_suite.print_errors();
+    test_suite.report_status()
 }

crates/tinywasm/tests/test-wasm-3.rs

@@ -1,5 +1,5 @@
 use crate::testsuite::util::*;
-use std::{borrow::Cow, collections::HashMap};
+use std::{borrow::Cow, collections::HashMap, fs::canonicalize, path::PathBuf};
 
 use super::TestSuite;
 use eyre::{eyre, Result};
@@ -71,12 +71,12 @@ impl ModuleRegistry {
 }
 
 impl TestSuite {
-    pub fn run_paths(&mut self, tests: &[&str]) -> Result<()> {
+    pub fn run_paths(&mut self, tests: &[PathBuf]) -> Result<()> {
         for file_name in tests {
             let group_wast = std::fs::read(file_name).expect("failed to read test wast");
             let file = TestFile {
                 contents: std::str::from_utf8(&group_wast).expect("failed to convert to utf8"),
-                name: file_name.to_string(),
+                name: canonicalize(file_name).expect("failed to canonicalize file name").to_string_lossy().to_string(),
                 parent: "(custom group)".into(),
             };
 
@@ -240,12 +240,12 @@ impl TestSuite {
                             Ok(_) => {
                                 // - skip "zero byte expected" as the magic number is not checked by wasmparser
                                 //   (Don't need to error on this, doesn't matter if it's malformed)
-                                // - skip "integer representation too long" as this has some false positives
+                                // - skip "integer representation too long" as this has some false positives on older tests
                                 if message == "zero byte expected" || message == "integer representation too long" {
                                     continue;
                                 }
 
-                                Err(eyre!("expected module to be malformed"))
+                                Err(eyre!("expected module to be malformed: {message}"))
                             }
                             Err(_) => Ok(()),
                         },
@@ -413,7 +413,17 @@ impl TestSuite {
 
                 AssertReturn { span, exec, results } => {
                     info!("AssertReturn: {:?}", exec);
-                    let expected = convert_wastret(results.into_iter())?;
+                    let expected = match convert_wastret(results.into_iter()) {
+                        Err(err) => {
+                            test_group.add_result(
+                                &format!("AssertReturn(unsupported-{i})"),
+                                span.linecol_in(wast_raw),
+                                Err(eyre!("failed to convert expected results: {:?}", err)),
+                            );
+                            continue;
+                        }
+                        Ok(expected) => expected,
+                    };
 
                     let invoke = match match exec {
                         wast::WastExecute::Wat(_) => Err(eyre!("wat not supported")),