Cache environment definitions

Signed-off-by: itowlson <[email protected]>

Author: itowlson

crates/build/src/lib.rs

@@ -20,6 +20,7 @@ pub async fn build(
     manifest_file: &Path,
     component_ids: &[String],
     skip_target_checks: bool,
+    cache_root: Option<PathBuf>,
 ) -> Result<()> {
     let build_info = component_build_configs(manifest_file)
         .await
@@ -67,6 +68,7 @@ pub async fn build(
         let errors = spin_environments::validate_application_against_environment_ids(
             &application,
             build_info.deployment_targets(),
+            cache_root.clone(),
         )
         .await?;
 
@@ -193,6 +195,6 @@ mod tests {
     #[tokio::test]
     async fn can_load_even_if_trigger_invalid() {
         let bad_trigger_file = test_data_root().join("bad_trigger.toml");
-        build(&bad_trigger_file, &[], true).await.unwrap();
+        build(&bad_trigger_file, &[], true, None).await.unwrap();
     }
 }

crates/environments/src/environment_definition.rs

@@ -7,13 +7,16 @@ use spin_manifest::schema::v2::TargetEnvironmentRef;
 const DEFAULT_REGISTRY: &str = "fermyon.com";
 
 /// Loads the given `TargetEnvironment` from a registry.
-pub async fn load_environment(env_id: &TargetEnvironmentRef) -> anyhow::Result<TargetEnvironment> {
+pub async fn load_environment(
+    env_id: &TargetEnvironmentRef,
+    cache: &spin_loader::cache::Cache,
+) -> anyhow::Result<TargetEnvironment> {
     match env_id {
         TargetEnvironmentRef::DefaultRegistry(package) => {
-            load_environment_from_registry(DEFAULT_REGISTRY, package).await
+            load_environment_from_registry(DEFAULT_REGISTRY, package, cache).await
         }
         TargetEnvironmentRef::Registry { registry, package } => {
-            load_environment_from_registry(registry, package).await
+            load_environment_from_registry(registry, package, cache).await
         }
         TargetEnvironmentRef::WitDirectory { path } => load_environment_from_dir(path),
     }
@@ -22,23 +25,32 @@ pub async fn load_environment(env_id: &TargetEnvironmentRef) -> anyhow::Result<T
 async fn load_environment_from_registry(
     registry: &str,
     env_id: &str,
+    cache: &spin_loader::cache::Cache,
 ) -> anyhow::Result<TargetEnvironment> {
     use futures_util::TryStreamExt;
 
+    let cache_file = cache.wit_path(registry, env_id);
+    if cache.wit_path(registry, env_id).exists() {
+        // Failure to read from cache is not fatal - fall through to origin.
+        if let Ok(bytes) = tokio::fs::read(cache_file).await {
+            return TargetEnvironment::from_package_bytes(env_id, bytes);
+        }
+    }
+
     let (pkg_name, pkg_ver) = env_id.split_once('@').with_context(|| format!("Failed to parse target environment {env_id} as package reference - is the target correct?"))?;
     let env_pkg_ref: wasm_pkg_loader::PackageRef = pkg_name
         .parse()
         .with_context(|| format!("Environment {pkg_name} is not a valid package name"))?;
 
-    let registry: wasm_pkg_loader::Registry = registry
+    let wkg_registry: wasm_pkg_loader::Registry = registry
         .parse()
         .with_context(|| format!("Registry {registry} is not a valid registry name"))?;
 
     // TODO: this requires wkg configuration which shouldn't be on users:
     // is there a better way to handle it?
     let mut wkg_config = wasm_pkg_loader::Config::global_defaults()
         .unwrap_or_else(|_| wasm_pkg_loader::Config::empty());
-    wkg_config.set_package_registry_override(env_pkg_ref, registry);
+    wkg_config.set_package_registry_override(env_pkg_ref, wkg_registry);
 
     let mut client = wasm_pkg_loader::Client::new(wkg_config);
 
@@ -64,7 +76,9 @@ async fn load_environment_from_registry(
         .with_context(|| format!("Failed to get {env_id} package data from registry"))?
         .to_vec();
 
-    TargetEnvironment::from_package_bytes(env_id.to_owned(), bytes)
+    _ = cache.write_wit(&bytes, registry, env_id).await; // Failure to cache is not fatal
+
+    TargetEnvironment::from_package_bytes(env_id, bytes)
 }
 
 fn load_environment_from_dir(path: &Path) -> anyhow::Result<TargetEnvironment> {
@@ -93,7 +107,7 @@ pub struct TargetEnvironment {
 }
 
 impl TargetEnvironment {
-    fn from_package_bytes(name: String, bytes: Vec<u8>) -> anyhow::Result<Self> {
+    fn from_package_bytes(name: &str, bytes: Vec<u8>) -> anyhow::Result<Self> {
         let decoded = wit_component::decode(&bytes)
             .with_context(|| format!("Failed to decode package for environment {name}"))?;
         let package_id = decoded.package();
@@ -107,7 +121,7 @@ impl TargetEnvironment {
             .clone();
 
         Ok(Self {
-            name,
+            name: name.to_owned(),
             decoded,
             package,
             package_id,

crates/environments/src/lib.rs

@@ -12,12 +12,17 @@ use spin_manifest::schema::v2::TargetEnvironmentRef;
 pub async fn validate_application_against_environment_ids(
     application: &ApplicationToValidate,
     env_ids: &[TargetEnvironmentRef],
+    cache_root: Option<std::path::PathBuf>,
 ) -> anyhow::Result<Vec<anyhow::Error>> {
     if env_ids.is_empty() {
         return Ok(Default::default());
     }
 
-    let envs = try_join_all(env_ids.iter().map(load_environment)).await?;
+    let cache = spin_loader::cache::Cache::new(cache_root)
+        .await
+        .context("Unable to create cache")?;
+
+    let envs = try_join_all(env_ids.iter().map(|e| load_environment(e, &cache))).await?;
     validate_application_against_environments(application, &envs).await
 }
 

crates/loader/src/cache.rs

@@ -14,6 +14,7 @@ const REGISTRY_CACHE_DIR: &str = "registry";
 const MANIFESTS_DIR: &str = "manifests";
 const WASM_DIR: &str = "wasm";
 const DATA_DIR: &str = "data";
+const WIT_DIR: &str = "wit";
 
 /// Cache for registry entities.
 #[derive(Debug)]
@@ -57,6 +58,13 @@ impl Cache {
         self.root.join(DATA_DIR)
     }
 
+    /// The (Wasm-encoded) WIT directory for the current cache.
+    /// This does not reuse `wasm_dir` because it cannot be addressed
+    /// by digest.
+    fn wit_dir(&self) -> PathBuf {
+        self.root.join(WIT_DIR)
+    }
+
     /// Return the path to a wasm file given its digest.
     pub fn wasm_file(&self, digest: impl AsRef<str>) -> Result<PathBuf> {
         // Check the expected wasm directory first; else check the data directory as a fallback.
@@ -100,6 +108,18 @@ impl Cache {
         Ok(())
     }
 
+    /// Write the contents in the cache's wit directory.
+    pub async fn write_wit(
+        &self,
+        bytes: impl AsRef<[u8]>,
+        registry: impl AsRef<str>,
+        name: impl AsRef<str>,
+    ) -> Result<()> {
+        self.ensure_wit_dir(registry.as_ref()).await?;
+        write_file(&self.wit_path(registry, name), bytes.as_ref()).await?;
+        Ok(())
+    }
+
     /// The path of contents in the cache's wasm directory, which may or may not exist.
     pub fn wasm_path(&self, digest: impl AsRef<str>) -> PathBuf {
         self.wasm_dir().join(safe_name(digest).as_ref())
@@ -110,12 +130,24 @@ impl Cache {
         self.data_dir().join(safe_name(digest).as_ref())
     }
 
+    /// The path of contents in the cache's wit directory, which may or may not exist.
+    /// This is keyed by registry and package name, not by digest, because digest
+    /// lookups slow down the `build` command and obviate the value of the cache:
+    /// therefore, it assumes that WITs are immutable once published, and that users
+    /// will manage the situation when this assumption is invalid.
+    pub fn wit_path(&self, registry: impl AsRef<str>, name: impl AsRef<str>) -> PathBuf {
+        self.wit_dir()
+            .join(safe_name(registry).as_ref())
+            .join(safe_name(name).as_ref())
+    }
+
     /// Ensure the expected configuration directories are found in the root.
     /// └── <configuration-root>
     ///     └── registry
     ///             └──manifests
     ///             └──wasm
     ///             └─-data
+    ///             └─-wit
     pub async fn ensure_dirs(&self) -> Result<()> {
         tracing::debug!("using cache root directory {}", self.root.display());
 
@@ -148,10 +180,30 @@ impl Cache {
                 .with_context(|| format!("failed to create assets directory `{}`", p.display()))?;
         }
 
+        let p = root.join(WIT_DIR);
+        if !p.is_dir() {
+            create_dir_all(&p)
+                .await
+                .with_context(|| format!("failed to create wit directory `{}`", p.display()))?;
+        }
+
         self.dirs_ensured_once.store(true, Ordering::Relaxed);
 
         Ok(())
     }
+
+    async fn ensure_wit_dir(&self, registry: impl AsRef<str>) -> Result<()> {
+        self.ensure_dirs().await?;
+
+        let p = self.wit_dir().join(safe_name(registry).as_ref());
+        if !p.is_dir() {
+            create_dir_all(&p)
+                .await
+                .with_context(|| format!("failed to create wit directory `{}`", p.display()))?;
+        }
+
+        Ok(())
+    }
 }
 
 #[cfg(windows)]

src/commands/build.rs

@@ -53,7 +53,13 @@ impl BuildCommand {
             spin_common::paths::find_manifest_file_path(self.app_source.as_ref())?;
         notify_if_nondefault_rel(&manifest_file, distance);
 
-        spin_build::build(&manifest_file, &self.component_id, self.skip_target_checks).await?;
+        spin_build::build(
+            &manifest_file,
+            &self.component_id,
+            self.skip_target_checks,
+            None,
+        )
+        .await?;
 
         if self.up {
             let mut cmd = UpCommand::parse_from(

src/commands/registry.rs

@@ -75,7 +75,7 @@ impl Push {
         notify_if_nondefault_rel(&app_file, distance);
 
         if self.build {
-            spin_build::build(&app_file, &[], false).await?;
+            spin_build::build(&app_file, &[], false, self.cache_dir.clone()).await?;
         }
 
         let annotations = if self.annotations.is_empty() {

src/commands/up.rs

@@ -191,7 +191,7 @@ impl UpCommand {
         }
 
         if self.build {
-            app_source.build().await?;
+            app_source.build(&self.cache_dir).await?;
         }
         let mut locked_app = self
             .load_resolved_app_source(resolved_app_source, &working_dir)

src/commands/up/app_source.rs

@@ -56,9 +56,9 @@ impl AppSource {
         }
     }
 
-    pub async fn build(&self) -> anyhow::Result<()> {
+    pub async fn build(&self, cache_root: &Option<PathBuf>) -> anyhow::Result<()> {
         match self {
-            Self::File(path) => spin_build::build(path, &[], false).await,
+            Self::File(path) => spin_build::build(path, &[], false, cache_root.clone()).await,
             _ => Ok(()),
         }
     }