Merge pull request #2377 from itowlson/override-download-cache-dir

More forgiving caching

Author: itowlson

crates/loader/src/cache.rs

@@ -2,7 +2,10 @@
 
 use anyhow::{ensure, Context, Result};
 
-use std::path::{Path, PathBuf};
+use std::{
+    path::PathBuf,
+    sync::atomic::{AtomicBool, Ordering},
+};
 
 use crate::fs::{create_dir_all, write_file};
 
@@ -17,6 +20,9 @@ const DATA_DIR: &str = "data";
 pub struct Cache {
     /// Root directory for the cache instance.
     root: PathBuf,
+    /// Whether the cache directories have been checked to exist (and
+    /// created if necessary).
+    dirs_ensured_once: AtomicBool,
 }
 
 impl Cache {
@@ -29,9 +35,11 @@ impl Cache {
                 .join(CONFIG_DIR),
         };
         let root = root.join(REGISTRY_CACHE_DIR);
-        Self::ensure_dirs(&root).await?;
 
-        Ok(Self { root })
+        Ok(Self {
+            root,
+            dirs_ensured_once: AtomicBool::new(false),
+        })
     }
 
     /// The manifests directory for the current cache.
@@ -80,12 +88,14 @@ impl Cache {
 
     /// Write the contents in the cache's wasm directory.
     pub async fn write_wasm(&self, bytes: impl AsRef<[u8]>, digest: impl AsRef<str>) -> Result<()> {
+        self.ensure_dirs().await?;
         write_file(&self.wasm_path(digest), bytes.as_ref()).await?;
         Ok(())
     }
 
     /// Write the contents in the cache's data directory.
     pub async fn write_data(&self, bytes: impl AsRef<[u8]>, digest: impl AsRef<str>) -> Result<()> {
+        self.ensure_dirs().await?;
         write_file(&self.data_path(digest), bytes.as_ref()).await?;
         Ok(())
     }
@@ -106,8 +116,16 @@ impl Cache {
     ///             └──manifests
     ///             └──wasm
     ///             └─-data
-    async fn ensure_dirs(root: &Path) -> Result<()> {
-        tracing::debug!("using cache root directory {}", root.display());
+    pub async fn ensure_dirs(&self) -> Result<()> {
+        tracing::debug!("using cache root directory {}", self.root.display());
+
+        // We don't care about ordering as this function is idempotent -
+        // we are using an Atomic only for interior mutability.
+        if self.dirs_ensured_once.load(Ordering::Relaxed) {
+            return Ok(());
+        }
+
+        let root = &self.root;
 
         let p = root.join(MANIFESTS_DIR);
         if !p.is_dir() {
@@ -130,12 +148,14 @@ impl Cache {
                 .with_context(|| format!("failed to create assets directory `{}`", p.display()))?;
         }
 
+        self.dirs_ensured_once.store(true, Ordering::Relaxed);
+
         Ok(())
     }
 }
 
 #[cfg(windows)]
-fn safe_name(digest: impl AsRef<str>) -> impl AsRef<Path> {
+fn safe_name(digest: impl AsRef<str>) -> impl AsRef<std::path::Path> {
     digest.as_ref().replace(':', "_")
 }
 

crates/loader/src/local.rs

@@ -241,6 +241,7 @@ impl LocalLoader {
         } else {
             let _loading_permit = self.file_loading_permits.acquire().await?;
 
+            self.cache.ensure_dirs().await?;
             let dest = self.cache.wasm_path(digest);
             verified_download(url, digest, &dest)
                 .await

src/commands/registry.rs

@@ -57,6 +57,10 @@ pub struct Push {
     /// This is a string whose format is defined by the registry standard, and generally consists of <registry>/<username>/<application-name>:<version>. E.g. ghcr.io/ogghead/spin-test-app:0.1.0
     #[clap()]
     pub reference: String,
+
+    /// Cache directory for downloaded registry data.
+    #[clap(long)]
+    pub cache_dir: Option<PathBuf>,
 }
 
 impl Push {
@@ -66,7 +70,7 @@ impl Push {
             spin_build::build(&app_file, &[]).await?;
         }
 
-        let mut client = spin_oci::Client::new(self.insecure, None).await?;
+        let mut client = spin_oci::Client::new(self.insecure, self.cache_dir.clone()).await?;
 
         let _spinner = create_dotted_spinner(2000, "Pushing app to the Registry".to_owned());
 
@@ -95,12 +99,16 @@ pub struct Pull {
     /// This is a string whose format is defined by the registry standard, and generally consists of <registry>/<username>/<application-name>:<version>. E.g. ghcr.io/ogghead/spin-test-app:0.1.0
     #[clap()]
     pub reference: String,
+
+    /// Cache directory for downloaded registry data.
+    #[clap(long)]
+    pub cache_dir: Option<PathBuf>,
 }
 
 impl Pull {
     /// Pull a Spin application from an OCI registry
     pub async fn run(self) -> Result<()> {
-        let mut client = spin_oci::Client::new(self.insecure, None).await?;
+        let mut client = spin_oci::Client::new(self.insecure, self.cache_dir.clone()).await?;
 
         let _spinner = create_dotted_spinner(2000, "Pulling app from the Registry".to_owned());
 

src/commands/up.rs

@@ -95,6 +95,10 @@ pub struct UpCommand {
     #[clap(long = "temp", alias = "tmp")]
     pub tmp: Option<PathBuf>,
 
+    /// Cache directory for downloaded components and assets.
+    #[clap(long)]
+    pub cache_dir: Option<PathBuf>,
+
     /// For local apps with directory mounts and no excluded files, mount them directly instead of using a temporary
     /// directory.
     ///
@@ -421,7 +425,7 @@ impl UpCommand {
             // TODO: We could make the `--help` experience a little faster if
             // we could fetch just the locked app JSON at this stage.
             AppSource::OciRegistry(reference) => {
-                let mut client = spin_oci::Client::new(self.insecure, None)
+                let mut client = spin_oci::Client::new(self.insecure, self.cache_dir.clone())
                     .await
                     .context("cannot create registry client")?;
 
@@ -448,7 +452,7 @@ impl UpCommand {
                 } else {
                     FilesMountStrategy::Copy(working_dir.join("assets"))
                 };
-                spin_loader::from_file(&manifest_path, files_mount_strategy, None)
+                spin_loader::from_file(&manifest_path, files_mount_strategy, self.cache_dir.clone())
                     .await
                     .with_context(|| {
                         format!(