Use Till's lockfile

Signed-off-by: itowlson <[email protected]>

Author: itowlson

crates/build/src/lib.rs

@@ -32,7 +32,7 @@ pub async fn build(
         })?;
     let app_dir = parent_dir(manifest_file)?;
 
-    let build_result = build_components(component_ids, build_info.components(), app_dir);
+    let build_result = build_components(component_ids, build_info.components(), &app_dir);
 
     // Emit any required warnings now, so that they don't bury any errors.
     if let Some(e) = build_info.load_error() {
@@ -69,6 +69,7 @@ pub async fn build(
             &application,
             build_info.deployment_targets(),
             cache_root.clone(),
+            &app_dir,
         )
         .await?;
 
@@ -87,7 +88,7 @@ pub async fn build(
 fn build_components(
     component_ids: &[String],
     components: Vec<ComponentBuildInfo>,
-    app_dir: PathBuf,
+    app_dir: &Path,
 ) -> Result<(), anyhow::Error> {
     let components_to_build = if component_ids.is_empty() {
         components
@@ -117,7 +118,7 @@ fn build_components(
 
     components_to_build
         .into_iter()
-        .map(|c| build_component(c, &app_dir))
+        .map(|c| build_component(c, app_dir))
         .collect::<Result<Vec<_>, _>>()?;
 
     terminal::step!("Finished", "building all Spin components");

crates/environments/src/environment_definition.rs

@@ -1,39 +1,117 @@
-use std::path::Path;
+use std::{collections::HashMap, path::Path};
 
 use anyhow::Context;
+use futures::future::try_join_all;
 use spin_common::ui::quoted_path;
 use spin_manifest::schema::v2::TargetEnvironmentRef;
 
 const DEFAULT_REGISTRY: &str = "fermyon.com";
 
-/// Loads the given `TargetEnvironment` from a registry.
-pub async fn load_environment(
+/// Serialisation format for the lockfile: registry -> { name -> digest }
+#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
+struct TargetEnvironmentLockfile(HashMap<String, HashMap<String, String>>);
+
+impl TargetEnvironmentLockfile {
+    fn digest(&self, registry: &str, env_id: &str) -> Option<&str> {
+        self.0
+            .get(registry)
+            .and_then(|m| m.get(env_id))
+            .map(|s| s.as_str())
+    }
+
+    fn set_digest(&mut self, registry: &str, env_id: &str, digest: &str) {
+        match self.0.get_mut(registry) {
+            Some(map) => {
+                map.insert(env_id.to_string(), digest.to_string());
+            }
+            None => {
+                let map = vec![(env_id.to_string(), digest.to_string())]
+                    .into_iter()
+                    .collect();
+                self.0.insert(registry.to_string(), map);
+            }
+        }
+    }
+}
+
+/// Load all the listed environments from their registries or paths.
+/// Registry data will be cached, with a lockfile under `.spin` mapping
+/// environment IDs to digests (to allow cache lookup without needing
+/// to fetch the digest from the registry).
+pub async fn load_environments(
+    env_ids: &[TargetEnvironmentRef],
+    cache_root: Option<std::path::PathBuf>,
+    app_dir: &std::path::Path,
+) -> anyhow::Result<Vec<TargetEnvironment>> {
+    if env_ids.is_empty() {
+        return Ok(Default::default());
+    }
+
+    let cache = spin_loader::cache::Cache::new(cache_root)
+        .await
+        .context("Unable to create cache")?;
+    let lockfile_dir = app_dir.join(".spin");
+    let lockfile_path = lockfile_dir.join("target-environments.lock");
+
+    let orig_lockfile: TargetEnvironmentLockfile = tokio::fs::read_to_string(&lockfile_path)
+        .await
+        .ok()
+        .and_then(|s| serde_json::from_str(&s).ok())
+        .unwrap_or_default();
+    let lockfile = std::sync::Arc::new(tokio::sync::RwLock::new(orig_lockfile.clone()));
+
+    let envs = try_join_all(
+        env_ids
+            .iter()
+            .map(|e| load_environment(e, &cache, &lockfile)),
+    )
+    .await?;
+
+    let final_lockfile = &*lockfile.read().await;
+    if *final_lockfile != orig_lockfile {
+        if let Ok(lockfile_json) = serde_json::to_string_pretty(&final_lockfile) {
+            _ = tokio::fs::create_dir_all(lockfile_dir).await;
+            _ = tokio::fs::write(&lockfile_path, lockfile_json).await; // failure to update lockfile is not an error
+        }
+    }
+
+    Ok(envs)
+}
+
+/// Loads the given `TargetEnvironment` from a registry or directory.
+async fn load_environment(
     env_id: &TargetEnvironmentRef,
     cache: &spin_loader::cache::Cache,
+    lockfile: &std::sync::Arc<tokio::sync::RwLock<TargetEnvironmentLockfile>>,
 ) -> anyhow::Result<TargetEnvironment> {
     match env_id {
         TargetEnvironmentRef::DefaultRegistry(package) => {
-            load_environment_from_registry(DEFAULT_REGISTRY, package, cache).await
+            load_environment_from_registry(DEFAULT_REGISTRY, package, cache, lockfile).await
         }
         TargetEnvironmentRef::Registry { registry, package } => {
-            load_environment_from_registry(registry, package, cache).await
+            load_environment_from_registry(registry, package, cache, lockfile).await
         }
         TargetEnvironmentRef::WitDirectory { path } => load_environment_from_dir(path),
     }
 }
 
+/// Loads the given `TargetEnvironment` from the given registry, or
+/// from cache if available. If the environment is not in cache, the
+/// encoded WIT will be cached, and the in-memory lockfile object
+/// updated.
 async fn load_environment_from_registry(
     registry: &str,
     env_id: &str,
     cache: &spin_loader::cache::Cache,
+    lockfile: &std::sync::Arc<tokio::sync::RwLock<TargetEnvironmentLockfile>>,
 ) -> anyhow::Result<TargetEnvironment> {
     use futures_util::TryStreamExt;
 
-    let cache_file = cache.wit_path(registry, env_id);
-    if cache.wit_path(registry, env_id).exists() {
-        // Failure to read from cache is not fatal - fall through to origin.
-        if let Ok(bytes) = tokio::fs::read(cache_file).await {
-            return TargetEnvironment::from_package_bytes(env_id, bytes);
+    if let Some(digest) = lockfile.read().await.digest(registry, env_id) {
+        if let Ok(cache_file) = cache.wasm_file(digest) {
+            if let Ok(bytes) = tokio::fs::read(&cache_file).await {
+                return TargetEnvironment::from_package_bytes(env_id, bytes);
+            }
         }
     }
 
@@ -76,7 +154,9 @@ async fn load_environment_from_registry(
         .with_context(|| format!("Failed to get {env_id} package data from registry"))?
         .to_vec();
 
-    _ = cache.write_wit(&bytes, registry, env_id).await; // Failure to cache is not fatal
+    let digest = release.content_digest.to_string();
+    _ = cache.write_wasm(&bytes, &digest).await; // Failure to cache is not fatal
+    lockfile.write().await.set_digest(registry, env_id, &digest);
 
     TargetEnvironment::from_package_bytes(env_id, bytes)
 }

crates/environments/src/lib.rs

@@ -3,8 +3,7 @@ use anyhow::{anyhow, Context};
 mod environment_definition;
 mod loader;
 
-use environment_definition::{load_environment, TargetEnvironment, TriggerType};
-use futures::future::try_join_all;
+use environment_definition::{load_environments, TargetEnvironment, TriggerType};
 pub use loader::ApplicationToValidate;
 use loader::ComponentToValidate;
 use spin_manifest::schema::v2::TargetEnvironmentRef;
@@ -13,16 +12,13 @@ pub async fn validate_application_against_environment_ids(
     application: &ApplicationToValidate,
     env_ids: &[TargetEnvironmentRef],
     cache_root: Option<std::path::PathBuf>,
+    app_dir: &std::path::Path,
 ) -> anyhow::Result<Vec<anyhow::Error>> {
     if env_ids.is_empty() {
         return Ok(Default::default());
     }
 
-    let cache = spin_loader::cache::Cache::new(cache_root)
-        .await
-        .context("Unable to create cache")?;
-
-    let envs = try_join_all(env_ids.iter().map(|e| load_environment(e, &cache))).await?;
+    let envs = load_environments(env_ids, cache_root, app_dir).await?;
     validate_application_against_environments(application, &envs).await
 }
 

crates/loader/src/cache.rs

@@ -14,7 +14,6 @@ const REGISTRY_CACHE_DIR: &str = "registry";
 const MANIFESTS_DIR: &str = "manifests";
 const WASM_DIR: &str = "wasm";
 const DATA_DIR: &str = "data";
-const WIT_DIR: &str = "wit";
 
 /// Cache for registry entities.
 #[derive(Debug)]
@@ -58,13 +57,6 @@ impl Cache {
         self.root.join(DATA_DIR)
     }
 
-    /// The (Wasm-encoded) WIT directory for the current cache.
-    /// This does not reuse `wasm_dir` because it cannot be addressed
-    /// by digest.
-    fn wit_dir(&self) -> PathBuf {
-        self.root.join(WIT_DIR)
-    }
-
     /// Return the path to a wasm file given its digest.
     pub fn wasm_file(&self, digest: impl AsRef<str>) -> Result<PathBuf> {
         // Check the expected wasm directory first; else check the data directory as a fallback.
@@ -108,18 +100,6 @@ impl Cache {
         Ok(())
     }
 
-    /// Write the contents in the cache's wit directory.
-    pub async fn write_wit(
-        &self,
-        bytes: impl AsRef<[u8]>,
-        registry: impl AsRef<str>,
-        name: impl AsRef<str>,
-    ) -> Result<()> {
-        self.ensure_wit_dir(registry.as_ref()).await?;
-        write_file(&self.wit_path(registry, name), bytes.as_ref()).await?;
-        Ok(())
-    }
-
     /// The path of contents in the cache's wasm directory, which may or may not exist.
     pub fn wasm_path(&self, digest: impl AsRef<str>) -> PathBuf {
         self.wasm_dir().join(safe_name(digest).as_ref())
@@ -130,24 +110,12 @@ impl Cache {
         self.data_dir().join(safe_name(digest).as_ref())
     }
 
-    /// The path of contents in the cache's wit directory, which may or may not exist.
-    /// This is keyed by registry and package name, not by digest, because digest
-    /// lookups slow down the `build` command and obviate the value of the cache:
-    /// therefore, it assumes that WITs are immutable once published, and that users
-    /// will manage the situation when this assumption is invalid.
-    pub fn wit_path(&self, registry: impl AsRef<str>, name: impl AsRef<str>) -> PathBuf {
-        self.wit_dir()
-            .join(safe_name(registry).as_ref())
-            .join(safe_name(name).as_ref())
-    }
-
     /// Ensure the expected configuration directories are found in the root.
     /// └── <configuration-root>
     ///     └── registry
     ///             └──manifests
     ///             └──wasm
     ///             └─-data
-    ///             └─-wit
     pub async fn ensure_dirs(&self) -> Result<()> {
         tracing::debug!("using cache root directory {}", self.root.display());
 
@@ -180,30 +148,10 @@ impl Cache {
                 .with_context(|| format!("failed to create assets directory `{}`", p.display()))?;
         }
 
-        let p = root.join(WIT_DIR);
-        if !p.is_dir() {
-            create_dir_all(&p)
-                .await
-                .with_context(|| format!("failed to create wit directory `{}`", p.display()))?;
-        }
-
         self.dirs_ensured_once.store(true, Ordering::Relaxed);
 
         Ok(())
     }
-
-    async fn ensure_wit_dir(&self, registry: impl AsRef<str>) -> Result<()> {
-        self.ensure_dirs().await?;
-
-        let p = self.wit_dir().join(safe_name(registry).as_ref());
-        if !p.is_dir() {
-            create_dir_all(&p)
-                .await
-                .with_context(|| format!("failed to create wit directory `{}`", p.display()))?;
-        }
-
-        Ok(())
-    }
 }
 
 #[cfg(windows)]