Permit more than one runtime

Signed-off-by: Ryan Levick <[email protected]>

Author: rylev

tests/runtime-tests/src/lib.rs

@@ -1,14 +1,14 @@
 use anyhow::Context;
 use std::path::{Path, PathBuf};
 use testing_framework::{
-    EnvTemplate, OnTestError, ServicesConfig, Spin, TestEnvironment, TestEnvironmentConfig,
-    TestError, TestResult,
+    EnvTemplate, InMemorySpin, OnTestError, Runtime, ServicesConfig, Spin, SpinConfig,
+    TestEnvironment, TestEnvironmentConfig, TestError, TestResult,
 };
 
 /// Configuration for a runtime test
-pub struct RuntimeTestConfig {
+pub struct RuntimeTestConfig<R: Runtime> {
     pub test_path: PathBuf,
-    pub spin_binary: PathBuf,
+    pub runtime_config: R::Config,
     pub on_error: OnTestError,
 }
 
@@ -43,18 +43,20 @@ impl RuntimeTest<Spin> {
 
             let config = RuntimeTestConfig {
                 test_path: test.path(),
-                spin_binary: spin_binary.clone(),
+                runtime_config: SpinConfig {
+                    binary_path: spin_binary.clone(),
+                },
                 on_error,
             };
-            RuntimeTest::bootstrap(config)?.run();
+            Self::bootstrap(config)?.run();
         }
         Ok(())
     }
 
-    pub fn bootstrap(config: RuntimeTestConfig) -> anyhow::Result<Self> {
+    pub fn bootstrap(config: RuntimeTestConfig<Spin>) -> anyhow::Result<Self> {
         log::info!("Testing: {}", config.test_path.display());
         let test_path_clone = config.test_path.to_owned();
-        let spin_binary = config.spin_binary.clone();
+        let spin_binary = config.runtime_config.binary_path.clone();
         let preboot = move |env: &mut TestEnvironment<Spin>| {
             copy_manifest(&test_path_clone, env)?;
             Ok(())
@@ -77,6 +79,91 @@ impl RuntimeTest<Spin> {
 
     /// Run an individual test
     pub fn run(&mut self) {
+        self.run_test(|env| {
+            let runtime = env.runtime_mut();
+            let request = testing_framework::Request::new(reqwest::Method::GET, "/");
+            let response = runtime.make_http_request(request)?;
+            if response.status() == 200 {
+                return Ok(());
+            }
+            if response.status() != 500 {
+                return Err(anyhow::anyhow!("Runtime tests are expected to return either either a 200 or a 500, but it returned a {}", response.status()).into());
+            }
+            let text = response
+                .text()
+                .context("could not get runtime test HTTP response")?;
+            if text.is_empty() {
+                let stderr = runtime.stderr();
+                return Err(anyhow::anyhow!("Runtime tests are expected to return a response body, but the response body was empty.\nstderr:\n{stderr}").into());
+            }
+
+            Err(TestError::Failure(RuntimeTestFailure {
+                error: text,
+                stderr: runtime.stderr().to_owned(),
+            }))
+        })
+    }
+}
+
+impl RuntimeTest<InMemorySpin> {
+    /// Run the runtime tests suite.
+    ///
+    /// Error represents an error in bootstrapping the tests. What happens on individual test failures
+    /// is controlled by `on_error`.
+    pub fn run_all(tests_path: &Path, on_error: OnTestError) -> anyhow::Result<()> {
+        for test in std::fs::read_dir(tests_path)
+            .with_context(|| format!("failed to read test directory '{}'", tests_path.display()))?
+        {
+            let test = test.context("I/O error reading entry from test directory")?;
+            if !test.file_type()?.is_dir() {
+                log::debug!(
+                    "Ignoring non-sub-directory in test directory: {}",
+                    test.path().display()
+                );
+                continue;
+            }
+
+            let config = RuntimeTestConfig {
+                test_path: test.path(),
+                runtime_config: (),
+                on_error,
+            };
+            Self::bootstrap(config)?.run();
+        }
+        Ok(())
+    }
+
+    pub fn bootstrap(config: RuntimeTestConfig<InMemorySpin>) -> anyhow::Result<Self> {
+        log::info!("Testing: {}", config.test_path.display());
+        let test_path_clone = config.test_path.to_owned();
+        let preboot = move |env: &mut TestEnvironment<InMemorySpin>| {
+            copy_manifest(&test_path_clone, env)?;
+            Ok(())
+        };
+        let services_config = services_config(&config)?;
+        let env_config = TestEnvironmentConfig::in_memory(services_config, preboot);
+        let env = TestEnvironment::up(env_config)?;
+        Ok(Self {
+            test_path: config.test_path,
+            on_error: config.on_error,
+            env,
+        })
+    }
+
+    fn run(&mut self) {
+        self.run_test(|env| {
+            let runtime = env.runtime_mut();
+            todo!()
+        })
+    }
+}
+
+impl<R> RuntimeTest<R> {
+    /// Run an individual test
+    pub(crate) fn run_test(
+        &mut self,
+        test: impl FnOnce(&mut TestEnvironment<R>) -> TestResult<RuntimeTestFailure>,
+    ) {
         let on_error = self.on_error;
         // macro which will look at `on_error` and do the right thing
         macro_rules! error {
@@ -141,7 +228,7 @@ impl RuntimeTest<Spin> {
     }
 }
 
-fn services_config(config: &RuntimeTestConfig) -> anyhow::Result<ServicesConfig> {
+fn services_config<R: Runtime>(config: &RuntimeTestConfig<R>) -> anyhow::Result<ServicesConfig> {
     let required_services = required_services(&config.test_path)?;
     let services_config = ServicesConfig::new(required_services)?;
     Ok(services_config)
@@ -179,30 +266,6 @@ fn copy_manifest<R>(test_dir: &Path, env: &mut TestEnvironment<R>) -> anyhow::Re
     Ok(())
 }
 
-fn test(env: &mut TestEnvironment<Spin>) -> TestResult<RuntimeTestFailure> {
-    let runtime = env.runtime_mut();
-    let request = testing_framework::Request::new(reqwest::Method::GET, "/");
-    let response = runtime.make_http_request(request)?;
-    if response.status() == 200 {
-        return Ok(());
-    }
-    if response.status() != 500 {
-        return Err(anyhow::anyhow!("Runtime tests are expected to return either either a 200 or a 500, but it returned a {}", response.status()).into());
-    }
-    let text = response
-        .text()
-        .context("could not get runtime test HTTP response")?;
-    if text.is_empty() {
-        let stderr = runtime.stderr();
-        return Err(anyhow::anyhow!("Runtime tests are expected to return a response body, but the response body was empty.\nstderr:\n{stderr}").into());
-    }
-
-    Err(TestError::Failure(RuntimeTestFailure {
-        error: text,
-        stderr: runtime.stderr().to_owned(),
-    }))
-}
-
 /// A runtime test failure
 struct RuntimeTestFailure {
     /// The error message returned by the runtime

tests/runtime-tests/src/main.rs

@@ -1,7 +1,7 @@
 use std::path::PathBuf;
 
 use runtime_tests::RuntimeTest;
-use testing_framework::OnTestError;
+use testing_framework::{OnTestError, Spin};
 
 fn main() -> anyhow::Result<()> {
     env_logger::init();
@@ -12,5 +12,5 @@ fn main() -> anyhow::Result<()> {
         .unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests"));
 
     let config = OnTestError::Log;
-    RuntimeTest::run_all(&tests_path, spin_binary_path, config)
+    RuntimeTest::<Spin>::run_all(&tests_path, spin_binary_path, config)
 }

tests/runtime.rs

@@ -6,7 +6,7 @@ mod runtime_tests {
     // generates individual tests for each one.
     test_codegen_macro::codegen_runtime_tests!(
         ignore: [
-            // This test is flacky. Often gets "Connection reset by peer" errors.
+            // This test is flaky. Often gets "Connection reset by peer" errors.
             // https://github.com/fermyon/spin/issues/2265
             "outbound-postgres"
         ]
@@ -15,10 +15,12 @@ mod runtime_tests {
     fn run(test_path: PathBuf) {
         let config = runtime_tests::RuntimeTestConfig {
             test_path,
-            spin_binary: env!("CARGO_BIN_EXE_spin").into(),
+            runtime_config: testing_framework::SpinConfig {
+                binary_path: env!("CARGO_BIN_EXE_spin").into(),
+            },
             on_error: testing_framework::OnTestError::Panic,
         };
-        runtime_tests::RuntimeTest::bootstrap(config)
+        runtime_tests::RuntimeTest::<testing_framework::Spin>::bootstrap(config)
             .expect("failed to bootstrap runtime tests tests")
             .run();
     }

tests/testing-framework/src/lib.rs

@@ -12,8 +12,8 @@ mod test_environment;
 
 pub use manifest_template::EnvTemplate;
 pub use services::ServicesConfig;
-pub use spin::{Request, Response, Spin, SpinMode};
-pub use test_environment::{TestEnvironment, TestEnvironmentConfig};
+pub use spin::{Request, Response, Spin, SpinConfig, SpinMode};
+pub use test_environment::{InMemorySpin, TestEnvironment, TestEnvironmentConfig};
 
 #[derive(Debug, Clone, Copy)]
 /// What to do when a test errors
@@ -26,6 +26,7 @@ pub enum OnTestError {
 
 /// A runtime which can be tested
 pub trait Runtime {
+    type Config;
     /// Return an error if one has occurred
     fn error(&mut self) -> anyhow::Result<()>;
 }

tests/testing-framework/src/spin.rs

@@ -244,6 +244,8 @@ impl Drop for Spin {
 }
 
 impl Runtime for Spin {
+    type Config = SpinConfig;
+
     fn error(&mut self) -> anyhow::Result<()> {
         if !matches!(self.io_mode, IoMode::None) && self.try_wait()?.is_some() {
             anyhow::bail!("Spin exited early: {}", self.stderr());
@@ -253,6 +255,10 @@ impl Runtime for Spin {
     }
 }
 
+pub struct SpinConfig {
+    pub binary_path: std::path::PathBuf,
+}
+
 fn kill_process(process: &mut std::process::Child) {
     #[cfg(windows)]
     {

tests/testing-framework/src/test_environment.rs

@@ -191,3 +191,28 @@ impl TestEnvironmentConfig<Spin> {
         }
     }
 }
+
+pub struct InMemorySpin;
+
+impl Runtime for InMemorySpin {
+    type Config = ();
+
+    fn error(&mut self) -> anyhow::Result<()> {
+        Ok(())
+    }
+}
+
+impl TestEnvironmentConfig<InMemorySpin> {
+    pub fn in_memory(
+        services_config: ServicesConfig,
+        preboot: impl FnOnce(&mut TestEnvironment<InMemorySpin>) -> anyhow::Result<()> + 'static,
+    ) -> Self {
+        Self {
+            services_config,
+            create_runtime: Box::new(|env| {
+                preboot(env)?;
+                Ok(InMemorySpin)
+            }),
+        }
+    }
+}