Move to new test-environment crate

rylev · kate-goldenring · commit 82a3fda7ce98 · 2024-06-25T08:44:56.000-07:00
Signed-off-by: Ryan Levick &lt;ryan.levick@fermyon.com&gt;
diff --git a/conformance-tests/Cargo.toml b/conformance-tests/Cargo.toml
@@ -9,5 +9,7 @@ homepage.workspace = true
 
 [dependencies]
 anyhow = "1.0"
-testing-framework = { git = "https://github.com/kate-goldenring/spin", branch = "shim-test-environment" }
-conformance-tests = { git = "https://github.com/fermyon/conformance-tests" }
+conformance-tests = { git = "https://github.com/fermyon/conformance-tests", branch = "testing-environment" }
+log = "0.4"
+nix = "0.26"
+test-environment = { git = "https://github.com/fermyon/conformance-tests", branch = "testing-environment" }
diff --git a/conformance-tests/README.md b/conformance-tests/README.md
@@ -18,7 +18,7 @@ Containerd must be configured to access the `containerd-shim-spin`:
 2. Configure containerd to add the Spin shim as a runtime by adding the following to `/etc/containerd/config.toml`:
     ```toml
     [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.spin]
-        runtime_type = "io.containerd.spin.v2"
+        runtime_type = "/usr/bin/containerd-shim-spin-v2"
     ```
 3. Restart containerd if it is running as a service
     ```sh
diff --git a/conformance-tests/src/main.rs b/conformance-tests/src/main.rs
@@ -1,88 +1,244 @@
-use testing_framework::runtimes::spin_containerd_shim::SpinShim;
+use std::{
+    path::{Path, PathBuf},
+    process::{Command, Stdio},
+};
+
+use anyhow::Context as _;
+use test_environment::{
+    http::{Request, Response},
+    io::OutputStream,
+    services::ServicesConfig,
+    Runtime, TestEnvironment, TestEnvironmentConfig,
+};
 
 fn main() {
     let tests_dir = conformance_tests::download_tests().unwrap();
+    let mut args = std::env::args().skip(1);
+    let spin_binary = args
+        .next()
+        .expect("expected first arg to be path to Spin binary");
+    let ctr_binary = args
+        .next()
+        .expect("expected second arg to be path to ctr binary");
+    let ctr_binary = Path::new(&ctr_binary);
 
     for test in conformance_tests::tests(&tests_dir).unwrap() {
-        let spin_binary = std::path::Path::new("/home/kagold/projects/temp/spin");
-        let ctr_binary = "/usr/bin/ctr".into();
+        let spin_binary = PathBuf::from(spin_binary.clone());
         // Just using TTL.sh until we decide where to host these (local registry, ghcr, etc)
         let oci_image = format!("ttl.sh/{}:72h", test.name);
-        let env_config = testing_framework::TestEnvironmentConfig::containerd_shim(
-            ctr_binary,
+        let env_config = SpinShim::config(
+            ctr_binary.into(),
             oci_image.clone(),
             move |e| {
                 // TODO: consider doing this outside the test environment since it is not a part of the runtime execution
                 e.copy_into(&test.manifest, "spin.toml")?;
                 e.copy_into(&test.component, test.component.file_name().unwrap())?;
-                SpinShim::regisry_push(spin_binary, &oci_image, e).unwrap();
+                SpinShim::regisry_push(&spin_binary, &oci_image, e).unwrap();
                 Ok(())
             },
-            testing_framework::ServicesConfig::none(),
+            test_environment::services::ServicesConfig::none(),
         );
-        std::thread::sleep(std::time::Duration::from_secs(60));
-        let mut env = testing_framework::TestEnvironment::up(env_config, |_| Ok(())).unwrap();
+        let mut env = TestEnvironment::up(env_config, |_| Ok(())).unwrap();
         let spin = env.runtime_mut();
         for invocation in test.config.invocations {
             let conformance_tests::config::Invocation::Http(invocation) = invocation;
-            let headers = invocation
+            invocation
                 .request
-                .headers
-                .iter()
-                .map(|h| (h.name.as_str(), h.value.as_str()))
-                .collect::<Vec<_>>();
-            let request = testing_framework::http::Request::full(
-                match invocation.request.method {
-                    conformance_tests::config::Method::GET => testing_framework::http::Method::GET,
-                    conformance_tests::config::Method::POST => {
-                        testing_framework::http::Method::POST
-                    }
-                },
-                &invocation.request.path,
-                &headers,
-                invocation.request.body,
-            );
-            let response = spin.make_http_request(request).unwrap();
-            let stderr = spin.stderr();
-            let body = String::from_utf8(response.body())
-                .unwrap_or_else(|_| String::from("invalid utf-8"));
-            assert_eq!(
-                response.status(),
-                invocation.response.status,
-                "request to Spin failed\nstderr:\n{stderr}\nbody:\n{body}",
-            );
+                .send(|request| spin.make_http_request(request))
+                .unwrap();
+        }
+        println!("test passed: {}", test.name);
+    }
+}
+
+struct SpinShim {
+    process: std::process::Child,
+    #[allow(dead_code)]
+    stdout: OutputStream,
+    stderr: OutputStream,
+    io_mode: IoMode,
+}
+
+/// `ctr run` invocations require an ID that is unique to all currently running instances. Since
+/// only one test runs at a time, we can reuse a constant ID.
+const CTR_RUN_ID: &str = "run-id";
+
+impl SpinShim {
+    pub fn config(
+        ctr_binary: PathBuf,
+        oci_image: String,
+        preboot: impl FnOnce(&mut TestEnvironment<SpinShim>) -> anyhow::Result<()> + 'static,
+        services_config: ServicesConfig,
+    ) -> TestEnvironmentConfig<SpinShim> {
+        TestEnvironmentConfig {
+            services_config,
+            create_runtime: Box::new(move |env| {
+                preboot(env)?;
+                SpinShim::image_pull(&ctr_binary, &oci_image)?;
+                SpinShim::start(&ctr_binary, env, &oci_image, CTR_RUN_ID)
+            }),
+        }
+    }
+
+    pub fn regisry_push<R>(
+        spin_binary_path: &Path,
+        image: &str,
+        env: &mut TestEnvironment<R>,
+    ) -> anyhow::Result<()> {
+        // TODO: consider enabling configuring a port
+        Command::new(spin_binary_path)
+            .args(["registry", "push"])
+            .arg(image)
+            .current_dir(env.path())
+            .output()
+            .context("failed to push spin app to registry with 'spin'")?;
+        // TODO: assess output
+        Ok(())
+    }
+
+    pub fn image_pull(ctr_binary_path: &Path, image: &str) -> anyhow::Result<()> {
+        // TODO: consider enabling configuring a port
+        Command::new(ctr_binary_path)
+            .args(["image", "pull"])
+            .arg(image)
+            .output()
+            .context("failed to pull spin app with 'ctr'")?;
+        // TODO: assess output
+        Ok(())
+    }
 
-            let mut actual_headers = response
-                .headers()
-                .iter()
-                .map(|(k, v)| (k.to_lowercase(), v.to_lowercase()))
-                .collect::<std::collections::HashMap<_, _>>();
-            for expected_header in invocation.response.headers {
-                let expected_name = expected_header.name.to_lowercase();
-                let expected_value = expected_header.value.map(|v| v.to_lowercase());
-                let actual_value = actual_headers.remove(&expected_name);
-                let Some(actual_value) = actual_value.as_deref() else {
-                    if expected_header.optional {
-                        continue;
-                    } else {
-                        panic!(
-                            "expected header {name} not found in response",
-                            name = expected_header.name
-                        )
-                    }
-                };
-                if let Some(expected_value) = expected_value {
-                    assert_eq!(actual_value, expected_value);
+    /// Start the Spin app using `ctr run`
+    /// Equivalent of `sudo ctr run --rm --net-host --runtime io.containerd.spin.v2 ttl.sh/myapp:48h ctr-run-id bogus-arg` for image `ttl.sh/myapp:48h` and run id `ctr-run-id`
+    pub fn start<R>(
+        ctr_binary_path: &Path,
+        env: &mut TestEnvironment<R>,
+        image: &str,
+        ctr_run_id: &str,
+    ) -> anyhow::Result<Self> {
+        let port = 80;
+        let mut ctr_cmd = std::process::Command::new(ctr_binary_path);
+        let child = ctr_cmd
+            .arg("run")
+            .args([
+                "--memory-limit",
+                "10000",
+                "--rm",
+                "--net-host",
+                "--runtime",
+                "io.containerd.spin.v2",
+            ])
+            .arg(image)
+            .arg(ctr_run_id)
+            .arg("bogus-arg")
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped());
+        for (key, value) in env.env_vars() {
+            child.env(key, value);
+        }
+        let mut child = child.spawn()?;
+        let stdout = OutputStream::new(child.stdout.take().unwrap());
+        let stderr = OutputStream::new(child.stderr.take().unwrap());
+        log::debug!("Awaiting shim binary to start up on port {port}...");
+        let mut spin = Self {
+            process: child,
+            stdout,
+            stderr,
+            io_mode: IoMode::Http(port),
+        };
+        let start = std::time::Instant::now();
+        loop {
+            match std::net::TcpStream::connect(format!("127.0.0.1:{port}")) {
+                Ok(_) => {
+                    log::debug!("Spin shim started on port {port}.");
+                    return Ok(spin);
+                }
+                Err(e) => {
+                    let stderr = spin.stderr.output_as_str().unwrap_or("<non-utf8>");
+                    log::trace!("Checking that the Spin server started returned an error: {e}");
+                    log::trace!("Current spin stderr = '{stderr}'");
                 }
             }
-            if !actual_headers.is_empty() {
-                panic!("unexpected headers: {actual_headers:?}");
+            if let Some(status) = spin.try_wait()? {
+                anyhow::bail!(
+                    "Shim exited early with status code {:?}\n{}{}",
+                    status.code(),
+                    spin.stdout.output_as_str().unwrap_or("<non-utf8>"),
+                    spin.stderr.output_as_str().unwrap_or("<non-utf8>")
+                );
             }
 
-            if let Some(expected_body) = invocation.response.body {
-                assert_eq!(body, expected_body);
+            if start.elapsed() > std::time::Duration::from_secs(2 * 60) {
+                break;
             }
+            std::thread::sleep(std::time::Duration::from_millis(50));
         }
-        println!("test passed: {}", test.name);
+        anyhow::bail!(
+            "`ctr run` did not start server or error after two minutes. stderr:\n\t{}",
+            spin.stderr.output_as_str().unwrap_or("<non-utf8>")
+        )
+    }
+
+    /// Make an HTTP request against Spin
+    ///
+    /// Will fail if Spin has already exited or if the io mode is not HTTP
+    pub fn make_http_request(&mut self, request: Request<'_, String>) -> anyhow::Result<Response> {
+        let IoMode::Http(port) = self.io_mode;
+        if let Some(status) = self.try_wait()? {
+            anyhow::bail!(
+                "make_http_request - shim exited early with status code {:?}",
+                status.code()
+            );
+        }
+        log::debug!("Connecting to HTTP server on port {port}...");
+        let response = request.send("localhost", port).unwrap();
+        log::debug!("Awaiting response from server");
+        if let Some(status) = self.try_wait()? {
+            anyhow::bail!("Spin exited early with status code {:?}", status.code());
+        }
+        println!("Response: {}", response.status());
+        Ok(response)
+    }
+
+    pub fn stderr(&mut self) -> &str {
+        self.stderr.output_as_str().unwrap_or("<non-utf8>")
+    }
+
+    fn try_wait(&mut self) -> std::io::Result<Option<std::process::ExitStatus>> {
+        self.process.try_wait()
+    }
+}
+
+impl Drop for SpinShim {
+    fn drop(&mut self) {
+        kill_process(&mut self.process);
+    }
+}
+
+impl Runtime for SpinShim {
+    fn error(&mut self) -> anyhow::Result<()> {
+        if self.try_wait()?.is_some() {
+            anyhow::bail!("Containerd shim spin exited early: {}", self.stderr());
+        }
+
+        Ok(())
+    }
+}
+
+/// How this instance is communicating with the outside world
+pub enum IoMode {
+    /// An http server is running on this port
+    Http(u16),
+}
+
+/// Helper function to kill a process
+fn kill_process(process: &mut std::process::Child) {
+    #[cfg(windows)]
+    {
+        let _ = process.kill();
+    }
+    #[cfg(not(windows))]
+    {
+        let pid = nix::unistd::Pid::from_raw(process.id() as i32);
+        let _ = nix::sys::signal::kill(pid, nix::sys::signal::SIGTERM);
     }
 }
