Merge pull request #2542 from fermyon/conformance-tests

Run conformance tests

Author: rylev

Cargo.lock

@@ -61,7 +61,9 @@ spin-locked-app = { path = "crates/locked-app" }
 spin-manifest = { path = "crates/manifest" }
 spin-oci = { path = "crates/oci" }
 spin-plugins = { path = "crates/plugins" }
-spin-telemetry = { path = "crates/telemetry", features = ["tracing-log-compat"] }
+spin-telemetry = { path = "crates/telemetry", features = [
+  "tracing-log-compat",
+] }
 spin-templates = { path = "crates/templates" }
 spin-trigger = { path = "crates/trigger" }
 spin-trigger-http = { path = "crates/trigger-http" }
@@ -97,6 +99,7 @@ redis = "0.24"
 runtime-tests = { path = "tests/runtime-tests" }
 test-components = { path = "tests/test-components" }
 test-codegen-macro = { path = "crates/test-codegen-macro" }
+test-environment = { git = "https://github.com/fermyon/conformance-tests" }
 
 [build-dependencies]
 cargo-target-dep = { git = "https://github.com/fermyon/cargo-target-dep", rev = "482f269eceb7b1a7e8fc618bf8c082dd24979cf1" }
@@ -117,16 +120,23 @@ llm-metal = ["llm", "spin-trigger-http/llm-metal"]
 llm-cublas = ["llm", "spin-trigger-http/llm-cublas"]
 
 [workspace]
-members = ["crates/*", "tests/runtime-tests", "tests/testing-framework"]
+members = [
+  "crates/*",
+  "tests/conformance-tests",
+  "tests/runtime-tests",
+  "tests/testing-framework",
+]
 
 [workspace.dependencies]
 anyhow = "1.0.75"
 http-body-util = "0.1.0"
 hyper = { version = "1.0.0", features = ["full"] }
-reqwest = { version = "0.11", features = ["stream", "blocking"] }
+reqwest = { version = "0.12", features = ["stream", "blocking"] }
 tracing = { version = "0.1", features = ["log"] }
 
-wasi-common-preview1 = { version = "21.0.1", package = "wasi-common", features = ["tokio"] }
+wasi-common-preview1 = { version = "21.0.1", package = "wasi-common", features = [
+  "tokio",
+] }
 wasmtime = "21.0.1"
 wasmtime-wasi = "21.0.1"
 wasmtime-wasi-http = "21.0.1"

Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "conformance"
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+homepage.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+
+[dependencies]
+anyhow = "1.0"
+testing-framework = { path = "../testing-framework" }
+conformance-tests = { git = "https://github.com/fermyon/conformance-tests" }
+test-environment = { git = "https://github.com/fermyon/conformance-tests" }
+
+[lints]
+workspace = true

tests/conformance-tests/Cargo.toml

@@ -0,0 +1,42 @@
+use testing_framework::runtimes::spin_cli::{SpinCli, SpinConfig};
+
+fn main() {
+    let spin_binary: std::path::PathBuf = std::env::args()
+        .nth(1)
+        .expect("expected first argument to be path to spin binary")
+        .into();
+    let tests_dir = conformance_tests::download_tests().unwrap();
+
+    for test in conformance_tests::tests(&tests_dir).unwrap() {
+        let env_config = SpinCli::config(
+            SpinConfig {
+                binary_path: spin_binary.clone(),
+                spin_up_args: Vec::new(),
+                app_type: testing_framework::runtimes::SpinAppType::Http,
+            },
+            test_environment::services::ServicesConfig::none(),
+            move |e| {
+                e.copy_into(&test.manifest, "spin.toml")?;
+                e.copy_into(&test.component, test.component.file_name().unwrap())?;
+                Ok(())
+            },
+        );
+        let mut env = test_environment::TestEnvironment::up(env_config, |_| Ok(())).unwrap();
+        let spin = env.runtime_mut();
+        for invocation in test.config.invocations {
+            let conformance_tests::config::Invocation::Http(invocation) = invocation;
+            let actual = invocation
+                .request
+                .send(|request| spin.make_http_request(request))
+                .unwrap();
+            if let Err(e) =
+                conformance_tests::assertions::assert_response(&invocation.response, &actual)
+            {
+                eprintln!("Test failed: {e}");
+                eprintln!("stderr: {}", spin.stderr());
+                std::process::exit(1);
+            }
+        }
+    }
+    println!("All tests passed!")
+}