loader: Update to manifest v2

Move spin-trigger UI tests to spin-loader.

Signed-off-by: Lann Martin <[email protected]>

Author: lann

crates/loader/Cargo.toml

@@ -22,18 +22,26 @@ reqwest = "0.11.9"
 semver = "1.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
+sha2 = "0.10.8"
 shellexpand = "3.1"
+spin-app = { path = "../app" }
 spin-common = { path = "../common" }
 spin-config = { path = "../config" }
 spin-manifest = { path = "../manifest" }
-tempfile = "3.3.0"
+tempfile = "3.8.0"
 terminal = { path = "../terminal" }
+thiserror = "1.0.49"
 tokio = { version = "1.23", features = [ "full" ] }
 tokio-util = "0.6"
-toml = "0.5"
+toml = "0.8.2"
 tracing = { workspace = true }
 walkdir = "2.3.2"
 
-[features]
-default = ["local"]
-local = []
+[dev-dependencies]
+tokio = { version = "1.23", features = ["rt", "macros"] }
+ui-testing = { path = "../ui-testing" }
+
+[[test]]
+name = "ui"
+path = "tests/ui.rs"
+harness = false

crates/loader/src/assets.rs

@@ -1,6 +1,6 @@
 //! Cache for OCI registry entities.
 
-use anyhow::{bail, Context, Result};
+use anyhow::{ensure, Context, Result};
 use tokio::fs;
 
 use std::path::{Path, PathBuf};
@@ -12,6 +12,7 @@ const WASM_DIR: &str = "wasm";
 const DATA_DIR: &str = "data";
 
 /// Cache for registry entities.
+#[derive(Debug)]
 pub struct Cache {
     /// Root directory for the cache instance.
     root: PathBuf,
@@ -49,40 +50,48 @@ impl Cache {
 
     /// Return the path to a wasm file given its digest.
     pub fn wasm_file(&self, digest: impl AsRef<str>) -> Result<PathBuf> {
-        let path = &self.wasm_dir().join(digest.as_ref());
-        match path.exists() {
-            true => Ok(path.into()),
-            false => bail!(format!(
-                "cannot find wasm file for digest {}",
-                digest.as_ref()
-            )),
-        }
+        let path = self.wasm_path(&digest);
+        ensure!(
+            path.exists(),
+            "cannot find wasm file for digest {}",
+            digest.as_ref()
+        );
+        Ok(path)
     }
 
     /// Return the path to a data file given its digest.
     pub fn data_file(&self, digest: impl AsRef<str>) -> Result<PathBuf> {
-        let path = &self.data_dir().join(digest.as_ref());
-        match path.exists() {
-            true => Ok(path.into()),
-            false => bail!(format!(
-                "cannot find data file for digest {}",
-                digest.as_ref()
-            )),
-        }
+        let path = self.data_path(&digest);
+        ensure!(
+            path.exists(),
+            "cannot find data file for digest {}",
+            digest.as_ref()
+        );
+        Ok(path)
     }
 
     /// Write the contents in the cache's wasm directory.
     pub async fn write_wasm(&self, bytes: impl AsRef<[u8]>, digest: impl AsRef<str>) -> Result<()> {
-        fs::write(self.wasm_dir().join(digest.as_ref()), bytes.as_ref()).await?;
+        fs::write(self.wasm_path(digest), bytes.as_ref()).await?;
         Ok(())
     }
 
     /// Write the contents in the cache's data directory.
     pub async fn write_data(&self, bytes: impl AsRef<[u8]>, digest: impl AsRef<str>) -> Result<()> {
-        fs::write(self.data_dir().join(digest.as_ref()), bytes.as_ref()).await?;
+        fs::write(self.data_path(digest), bytes.as_ref()).await?;
         Ok(())
     }
 
+    /// The path of contents in the cache's wasm directory, which may or may not exist.
+    pub fn wasm_path(&self, digest: impl AsRef<str>) -> PathBuf {
+        self.wasm_dir().join(digest.as_ref())
+    }
+
+    /// The path of contents in the cache's wasm directory, which may or may not exist.
+    pub fn data_path(&self, digest: impl AsRef<str>) -> PathBuf {
+        self.data_dir().join(digest.as_ref())
+    }
+
     /// Ensure the expected configuration directories are found in the root.
     /// └── <configuration-root>
     ///     └── registry

crates/loader/src/cache.rs

@@ -0,0 +1,44 @@
+use std::path::Path;
+
+use anyhow::{ensure, Context, Result};
+use sha2::Digest;
+use tokio::io::AsyncWriteExt;
+
+/// Download content from `url` while will be verified to match `digest` and
+/// then moved to `dest`.
+pub async fn verified_download(url: &str, digest: &str, dest: &Path) -> Result<()> {
+    tracing::debug!("Downloading content from {url:?}");
+
+    // Prepare tempfile destination
+    let prefix = format!("download-{}", digest.replace(':', "-"));
+    let dest_dir = dest.parent().context("invalid dest")?;
+    let (temp_file, temp_path) = tempfile::NamedTempFile::with_prefix_in(prefix, dest_dir)
+        .context("error creating download tempfile")?
+        .into_parts();
+
+    // Begin download
+    let mut resp = reqwest::get(url).await?.error_for_status()?;
+
+    // Hash as we write to the tempfile
+    let mut hasher = sha2::Sha256::new();
+    {
+        let mut temp_file = tokio::fs::File::from_std(temp_file);
+        while let Some(chunk) = resp.chunk().await? {
+            hasher.update(&chunk);
+            temp_file.write_all(&chunk).await?;
+        }
+        temp_file.flush().await?;
+    }
+
+    // Check the digest
+    let actual_digest = format!("sha256:{:x}", hasher.finalize());
+    ensure!(
+        actual_digest == digest,
+        "invalid content digest; expected {digest}, downloaded {actual_digest}"
+    );
+
+    // Move to final destination
+    temp_path.persist_noclobber(dest)?;
+
+    Ok(())
+}

crates/loader/src/common.rs

@@ -10,18 +10,40 @@
 
 #![deny(missing_docs)]
 
-mod assets;
+use std::path::{Path, PathBuf};
+
+use anyhow::Result;
+use local::LocalLoader;
+use spin_app::locked::LockedApp;
+use spin_common::paths::parent_dir;
+
 pub mod cache;
-mod common;
-#[cfg(feature = "local")]
-pub mod local;
-mod validation;
+mod http;
+mod local;
 
-/// Load a Spin application configuration from a spin.toml manifest file.
-#[cfg(feature = "local")]
-pub use local::from_file;
+/// Maximum number of files to copy (or download) concurrently
+pub(crate) const MAX_FILE_LOADING_CONCURRENCY: usize = 16;
 
-/// Maximum number of assets to process in parallel
-pub(crate) const MAX_PARALLEL_ASSET_PROCESSING: usize = 16;
+/// Load a Spin locked app from a spin.toml manifest file. If `files_mount_root`
+/// is given, `files` mounts will be copied to that directory. If not, `files`
+/// mounts will validated as "direct mounts".
+pub async fn from_file(
+    manifest_path: impl AsRef<Path>,
+    files_mount_strategy: FilesMountStrategy,
+) -> Result<LockedApp> {
+    let path = manifest_path.as_ref();
+    let app_root = parent_dir(path)?;
+    let loader = LocalLoader::new(&app_root, files_mount_strategy).await?;
+    loader.load_file(path).await
+}
 
-pub use assets::to_relative;
+/// The strategy to use for mounting WASI files into a guest.
+#[derive(Debug)]
+pub enum FilesMountStrategy {
+    /// Copy files into the given mount root directory.
+    Copy(PathBuf),
+    /// Mount files directly from their source director(ies). This only
+    /// supports mounting full directories; mounting single files, glob
+    /// patterns, and `exclude_files` are not supported.
+    Direct,
+}