Merge pull request #2591 from fermyon/conformance-tests-update2

Conformance Tests Update

Author: rylev

Cargo.lock

@@ -99,7 +99,7 @@ redis = "0.24"
 runtime-tests = { path = "tests/runtime-tests" }
 test-components = { path = "tests/test-components" }
 test-codegen-macro = { path = "crates/test-codegen-macro" }
-test-environment = { git = "https://github.com/fermyon/conformance-tests" }
+test-environment = { git = "https://github.com/fermyon/conformance-tests", branch = "main" }
 
 [build-dependencies]
 cargo-target-dep = { git = "https://github.com/fermyon/cargo-target-dep", rev = "482f269eceb7b1a7e8fc618bf8c082dd24979cf1" }

Cargo.toml

@@ -11,8 +11,8 @@ rust-version.workspace = true
 [dependencies]
 anyhow = "1.0"
 testing-framework = { path = "../testing-framework" }
-conformance-tests = { git = "https://github.com/fermyon/conformance-tests" }
-test-environment = { git = "https://github.com/fermyon/conformance-tests" }
+conformance-tests = { git = "https://github.com/fermyon/conformance-tests", branch = "main" }
+test-environment = { git = "https://github.com/fermyon/conformance-tests", branch = "main" }
 
 [lints]
 workspace = true

tests/conformance-tests/Cargo.toml

@@ -1,48 +1,60 @@
+use anyhow::Context as _;
 use testing_framework::runtimes::spin_cli::{SpinCli, SpinConfig};
 
 fn main() {
     let spin_binary: std::path::PathBuf = std::env::args()
         .nth(1)
         .expect("expected first argument to be path to spin binary")
         .into();
-    let tests_dir = conformance_tests::download_tests().unwrap();
+    conformance_tests::run_tests(move |test| run_test(test, &spin_binary)).unwrap();
+}
 
-    for test in conformance_tests::tests(&tests_dir).unwrap() {
-        println!("Running test '{}'", test.name);
-        let env_config = SpinCli::config(
-            SpinConfig {
-                binary_path: spin_binary.clone(),
-                spin_up_args: Vec::new(),
-                app_type: testing_framework::runtimes::SpinAppType::Http,
-            },
-            test_environment::services::ServicesConfig::new(test.config.services).unwrap(),
-            move |e| {
-                let mut manifest =
-                    test_environment::manifest_template::EnvTemplate::from_file(&test.manifest)
-                        .unwrap();
-                manifest.substitute(e, |_| None).unwrap();
-                e.write_file("spin.toml", manifest.contents())?;
-                e.copy_into(&test.component, test.component.file_name().unwrap())?;
-                Ok(())
-            },
-        );
-        let mut env = test_environment::TestEnvironment::up(env_config, |_| Ok(())).unwrap();
-        for invocation in test.config.invocations {
-            let conformance_tests::config::Invocation::Http(mut invocation) = invocation;
-            invocation.request.substitute_from_env(&mut env).unwrap();
-            let spin = env.runtime_mut();
-            let actual = invocation
-                .request
-                .send(|request| spin.make_http_request(request))
-                .unwrap();
-            if let Err(e) =
-                conformance_tests::assertions::assert_response(&invocation.response, &actual)
-            {
-                eprintln!("Test '{}' failed: {e}", test.name);
-                eprintln!("stderr: {}", spin.stderr());
-                std::process::exit(1);
+fn run_test(test: conformance_tests::Test, spin_binary: &std::path::Path) -> anyhow::Result<()> {
+    let mut services = Vec::new();
+    for precondition in test.config.preconditions {
+        match precondition {
+            conformance_tests::config::Precondition::HttpEcho => {
+                services.push("http-echo".into());
+            }
+            conformance_tests::config::Precondition::TcpEcho => {
+                services.push("tcp-echo".into());
             }
+            conformance_tests::config::Precondition::KeyValueStore(_) => {}
         }
     }
-    println!("All tests passed!")
+    let env_config = SpinCli::config(
+        SpinConfig {
+            binary_path: spin_binary.to_owned(),
+            spin_up_args: Vec::new(),
+            app_type: testing_framework::runtimes::SpinAppType::Http,
+        },
+        test_environment::services::ServicesConfig::new(services)?,
+        move |e| {
+            let mut manifest =
+                test_environment::manifest_template::EnvTemplate::from_file(&test.manifest)?;
+            manifest.substitute(e, |_| None)?;
+            e.write_file("spin.toml", manifest.contents())?;
+            e.copy_into(&test.component, test.component.file_name().unwrap())?;
+            Ok(())
+        },
+    );
+    let mut env = test_environment::TestEnvironment::up(env_config, |_| Ok(()))?;
+    for invocation in test.config.invocations {
+        let conformance_tests::config::Invocation::Http(mut invocation) = invocation;
+        invocation.request.substitute_from_env(&mut env)?;
+        let spin = env.runtime_mut();
+        let actual = invocation
+            .request
+            .send(|request| spin.make_http_request(request))?;
+
+        conformance_tests::assertions::assert_response(&invocation.response, &actual)
+            .with_context(|| {
+                format!(
+                    "Failed assertion.\nstdout: {}\nstderr: {}",
+                    spin.stdout().to_owned(),
+                    spin.stderr()
+                )
+            })?;
+    }
+    Ok(())
 }

tests/conformance-tests/src/main.rs

@@ -13,5 +13,5 @@ anyhow = "1.0"
 env_logger = "0.10.0"
 log = "0.4"
 testing-framework = { path = "../testing-framework" }
-test-environment = { git = "https://github.com/fermyon/conformance-tests" }
+test-environment = { git = "https://github.com/fermyon/conformance-tests", branch = "main" }
 test-components = { path = "../test-components" }

tests/runtime-tests/Cargo.toml

@@ -13,7 +13,7 @@ nix = "0.26.1"
 regex = "1.10.2"
 reqwest = { workspace = true }
 temp-dir = "0.1.11"
-test-environment = { git = "https://github.com/fermyon/conformance-tests" }
+test-environment = { git = "https://github.com/fermyon/conformance-tests", branch = "main" }
 spin-trigger-http = { path = "../../crates/trigger-http" }
 spin-http = { path = "../../crates/http" }
 spin-trigger = { path = "../../crates/trigger" }