Merge pull request #2872 from itowlson/fix-runtime-test-failures-being-buried

Fix for failing runtime tests not failing the test suite

Author: rylev

Cargo.lock

@@ -119,7 +119,7 @@ members = [
 anyhow = "1"
 async-trait = "0.1"
 bytes = "1"
-conformance-tests = { git = "https://github.com/fermyon/conformance-tests", rev = "387b7f375df59e6254a7c29cf4a53507a9f46d32" }
+conformance-tests = { git = "https://github.com/fermyon/conformance-tests", rev = "ecd22a45bcc5c775a56c67689a89aa4005866ac0" }
 dirs = "5.0"
 futures = "0.3"
 glob = "0.3"
@@ -139,7 +139,7 @@ serde = { version = "1", features = ["derive", "rc"] }
 serde_json = "1.0"
 sha2 = "0.10"
 tempfile = "3"
-test-environment = { git = "https://github.com/fermyon/conformance-tests", rev = "387b7f375df59e6254a7c29cf4a53507a9f46d32" }
+test-environment = { git = "https://github.com/fermyon/conformance-tests", rev = "ecd22a45bcc5c775a56c67689a89aa4005866ac0" }
 thiserror = "1"
 tokio = "1"
 toml = "0.8"

Cargo.toml

@@ -3,8 +3,10 @@ fn main() {
         .nth(1)
         .expect("expected first argument to be path to spin binary")
         .into();
-    conformance_tests::run_tests("canary", move |test| {
+    let config = conformance_tests::Config::new("canary");
+    conformance_tests::run_tests(config, move |test| {
         conformance::run_test(test, &spin_binary)
     })
-    .unwrap();
+    .unwrap()
+    .exit();
 }

tests/conformance-tests/src/main.rs

@@ -27,11 +27,15 @@ mod runtime_tests {
     }
 
     #[test]
-    fn conformance_tests() {
-        conformance_tests::run_tests("canary", move |test| {
+    fn conformance_tests() -> anyhow::Result<()> {
+        let config = conformance_tests::Config::new("canary");
+        let conclusion = conformance_tests::run_tests(config, move |test| {
             conformance::run_test(test, &spin_binary())
-        })
-        .unwrap();
+        })?;
+        if conclusion.has_failed() {
+            anyhow::bail!("One or more errors occurred in the conformance tests");
+        }
+        Ok(())
     }
 
     fn spin_binary() -> PathBuf {