rejigger test generation and suites

fix the build and macros, separate wasi test suite from "command" test
suite, test suite for preview 1 should always panic

Author: Pat Hickey

Cargo.lock

@@ -8,6 +8,7 @@ members = [
     "verify",
     "host",
     "test-programs",
+    "test-programs/command-tests",
     "test-programs/wasi-tests",
     "test-programs/reactor-tests",
     "wasi-common",

Cargo.toml

@@ -18,7 +18,7 @@ tokio = { version = "1", features = ["full"] }
 [dev-dependencies]
 tracing = { workspace = true }
 cap-rand = { workspace = true }
-test-programs-macros = { path = "../test-programs/macros" }
+test-programs = { path = "../test-programs" }
 test-log = { version = "0.2", default-features = false, features = ["trace"] }
 tempfile = "3.3.0"
 lazy_static = "1"

host/Cargo.toml

@@ -31,18 +31,7 @@ lazy_static::lazy_static! {
     };
 }
 // uses ENGINE, creates a fn get_component(&str) -> Component
-test_programs_macros::command_components!();
-
-// A bunch of these test cases are expected to fail. We wrap up their execution in this
-// function so that we see if changes make them start passing.
-// Note that we need to be careful not to check in any tests that panic for this approach
-// to work.
-fn expect_fail(r: Result<()>) -> Result<()> {
-    match r {
-        Ok(_) => Err(anyhow::anyhow!("expected failure")),
-        Err(_) => Ok(()),
-    }
-}
+test_programs::command_tests_components!();
 
 struct CommandCtx {
     table: Table,
@@ -436,262 +425,6 @@ async fn default_clocks() -> Result<()> {
         .map_err(|()| anyhow::anyhow!("command returned with failing exit status"))
 }
 
-async fn run_with_temp_dir(component: &str) -> Result<()> {
-    let mut builder = WasiCtxBuilder::new();
-
-    if cfg!(windows) {
-        builder = builder
-            .push_env("ERRNO_MODE_WINDOWS", "1")
-            .push_env("NO_FDFLAGS_SYNC_SUPPORT", "1")
-            .push_env("NO_DANGLING_FILESYSTEM", "1")
-            .push_env("NO_RENAME_DIR_TO_EMPTY_DIR", "1");
-    }
-    if cfg!(all(unix, not(target_os = "macos"))) {
-        builder = builder.push_env("ERRNO_MODE_UNIX", "1");
-    }
-    if cfg!(target_os = "macos") {
-        builder = builder.push_env("ERRNO_MODE_MACOS", "1");
-    }
-
-    let dir = tempfile::tempdir()?;
-    let open_dir = Dir::open_ambient_dir(dir.path(), ambient_authority())?;
-
-    let mut table = Table::new();
-    let wasi = builder
-        .preopened_dir(open_dir, "/foo")
-        .set_args(&["program", "/foo"])
-        .build(&mut table)?;
-
-    let (mut store, command) =
-        instantiate(get_component(component), CommandCtx { table, wasi }).await?;
-
-    command
-        .call_main(&mut store)
-        .await?
-        .map_err(|()| anyhow::anyhow!("command returned with failing exit status"))
-}
-
-#[test_log::test(tokio::test)]
-async fn big_random_buf() -> Result<()> {
-    run_with_temp_dir("big_random_buf").await
-}
-
-#[test_log::test(tokio::test)]
-async fn clock_time_get() -> Result<()> {
-    run_with_temp_dir("clock_time_get").await
-}
-
-#[test_log::test(tokio::test)]
-async fn close_preopen() -> Result<()> {
-    run_with_temp_dir("close_preopen").await
-}
-
-#[test_log::test(tokio::test)]
-async fn overwrite_preopen() -> Result<()> {
-    run_with_temp_dir("overwrite_preopen").await
-}
-
-#[test_log::test(tokio::test)]
-async fn dangling_fd() -> Result<()> {
-    run_with_temp_dir("dangling_fd").await
-}
-
-#[test_log::test(tokio::test)]
-async fn dangling_symlink() -> Result<()> {
-    run_with_temp_dir("dangling_symlink").await
-}
-
-#[test_log::test(tokio::test)]
-async fn directory_seek() -> Result<()> {
-    run_with_temp_dir("directory_seek").await
-}
-
-#[test_log::test(tokio::test)]
-async fn fd_advise() -> Result<()> {
-    run_with_temp_dir("fd_advise").await
-}
-
-#[test_log::test(tokio::test)]
-async fn fd_filestat_get() -> Result<()> {
-    run_with_temp_dir("fd_filestat_get").await
-}
-
-#[test_log::test(tokio::test)]
-async fn fd_filestat_set() -> Result<()> {
-    run_with_temp_dir("fd_filestat_set").await
-}
-
-#[test_log::test(tokio::test)]
-async fn fd_flags_set() -> Result<()> {
-    run_with_temp_dir("fd_flags_set").await
-}
-
-#[test_log::test(tokio::test)]
-async fn fd_readdir() -> Result<()> {
-    run_with_temp_dir("fd_readdir").await
-}
-
-#[test_log::test(tokio::test)]
-async fn file_allocate() -> Result<()> {
-    run_with_temp_dir("file_allocate").await
-}
-
-#[test_log::test(tokio::test)]
-async fn file_pread_pwrite() -> Result<()> {
-    run_with_temp_dir("file_pread_pwrite").await
-}
-
-#[test_log::test(tokio::test)]
-async fn file_seek_tell() -> Result<()> {
-    run_with_temp_dir("file_seek_tell").await
-}
-
-#[test_log::test(tokio::test)]
-async fn file_truncation() -> Result<()> {
-    run_with_temp_dir("file_truncation").await
-}
-
-#[test_log::test(tokio::test)]
-async fn file_unbuffered_write() -> Result<()> {
-    run_with_temp_dir("file_unbuffered_write").await
-}
-
-#[test_log::test(tokio::test)]
-async fn interesting_paths() -> Result<()> {
-    if cfg!(windows) {
-        expect_fail(run_with_temp_dir("interesting_paths").await)
-    } else {
-        run_with_temp_dir("interesting_paths").await
-    }
-}
-
-#[test_log::test(tokio::test)]
-async fn isatty() -> Result<()> {
-    run_with_temp_dir("isatty").await
-}
-
-#[test_log::test(tokio::test)]
-async fn nofollow_errors() -> Result<()> {
-    run_with_temp_dir("nofollow_errors").await
-}
-
-#[test_log::test(tokio::test)]
-async fn path_exists() -> Result<()> {
-    run_with_temp_dir("path_exists").await
-}
-
-#[test_log::test(tokio::test)]
-async fn path_filestat() -> Result<()> {
-    run_with_temp_dir("path_filestat").await
-}
-
-#[test_log::test(tokio::test)]
-async fn path_link() -> Result<()> {
-    run_with_temp_dir("path_link").await
-}
-
-#[test_log::test(tokio::test)]
-async fn path_open_create_existing() -> Result<()> {
-    run_with_temp_dir("path_open_create_existing").await
-}
-
-#[test_log::test(tokio::test)]
-async fn path_open_dirfd_not_dir() -> Result<()> {
-    run_with_temp_dir("path_open_dirfd_not_dir").await
-}
-
-#[test_log::test(tokio::test)]
-async fn path_open_missing() -> Result<()> {
-    run_with_temp_dir("path_open_missing").await
-}
-
-#[test_log::test(tokio::test)]
-async fn path_rename() -> Result<()> {
-    run_with_temp_dir("path_rename").await
-}
-
-#[test_log::test(tokio::test)]
-async fn path_rename_dir_trailing_slashes() -> Result<()> {
-    run_with_temp_dir("path_rename_dir_trailing_slashes").await
-}
-
-#[test_log::test(tokio::test)]
-async fn path_rename_file_trailing_slashes() -> Result<()> {
-    // renaming a file with trailing slash in destination name expected to fail, but succeeds: line 18
-    expect_fail(run_with_temp_dir("path_rename_file_trailing_slashes").await)
-}
-
-#[test_log::test(tokio::test)]
-async fn path_symlink_trailing_slashes() -> Result<()> {
-    run_with_temp_dir("path_symlink_trailing_slashes").await
-}
-
-#[test_log::test(tokio::test)]
-async fn poll_oneoff_files() -> Result<()> {
-    if cfg!(windows) {
-        expect_fail(run_with_temp_dir("poll_oneoff_files").await)
-    } else {
-        run_with_temp_dir("poll_oneoff_files").await
-    }
-}
-
-#[test_log::test(tokio::test)]
-async fn poll_oneoff_stdio() -> Result<()> {
-    expect_fail(run_with_temp_dir("poll_oneoff_stdio").await)
-}
-
-#[test_log::test(tokio::test)]
-async fn readlink() -> Result<()> {
-    run_with_temp_dir("readlink").await
-}
-
-#[test_log::test(tokio::test)]
-async fn remove_directory_trailing_slashes() -> Result<()> {
-    // removing a directory with a trailing slash in the path succeeded under preview 1,
-    // fails now returning INVAL
-    expect_fail(run_with_temp_dir("remove_directory_trailing_slashes").await)
-}
-
-#[test_log::test(tokio::test)]
-async fn remove_nonempty_directory() -> Result<()> {
-    run_with_temp_dir("remove_nonempty_directory").await
-}
-
-#[test_log::test(tokio::test)]
-async fn renumber() -> Result<()> {
-    run_with_temp_dir("renumber").await
-}
-
-#[test_log::test(tokio::test)]
-async fn sched_yield() -> Result<()> {
-    run_with_temp_dir("sched_yield").await
-}
-
-#[test_log::test(tokio::test)]
-async fn stdio() -> Result<()> {
-    run_with_temp_dir("stdio").await
-}
-
-#[test_log::test(tokio::test)]
-async fn symlink_create() -> Result<()> {
-    run_with_temp_dir("symlink_create").await
-}
-
-#[test_log::test(tokio::test)]
-async fn symlink_filestat() -> Result<()> {
-    run_with_temp_dir("symlink_filestat").await
-}
-
-#[test_log::test(tokio::test)]
-async fn symlink_loop() -> Result<()> {
-    run_with_temp_dir("symlink_loop").await
-}
-
-#[test_log::test(tokio::test)]
-async fn unlink_file_trailing_slashes() -> Result<()> {
-    run_with_temp_dir("unlink_file_trailing_slashes").await
-}
-
 #[test_log::test(tokio::test)]
 async fn export_cabi_realloc() -> Result<()> {
     let mut table = Table::new();
@@ -734,8 +467,3 @@ async fn read_only() -> Result<()> {
         .await?
         .map_err(|()| anyhow::anyhow!("command returned with failing exit status"))
 }
-
-#[test_log::test(tokio::test)]
-async fn dir_fd_op_failures() -> Result<()> {
-    run_with_temp_dir("dir_fd_op_failures").await
-}

host/tests/command.rs

@@ -20,7 +20,7 @@ lazy_static::lazy_static! {
 }
 
 // uses ENGINE, creates a fn get_component(&str) -> Component
-test_programs_macros::reactor_components!();
+test_programs::reactor_tests_components!();
 
 wasmtime::component::bindgen!({
     path: "../test-programs/reactor-tests/wit",