Merge pull request #2957 from itowlson/templates-install-from-remote-tar

Install templates from remote tarball

Author: itowlson

Cargo.lock

@@ -6,9 +6,11 @@ edition = { workspace = true }
 
 [dependencies]
 anyhow = { workspace = true }
+bytes = { workspace = true }
 dialoguer = "0.11"
 fs_extra = "1"
 heck = "0.5"
+flate2 = "1"
 indexmap = { version = "2", features = ["serde"] }
 itertools = { workspace = true }
 lazy_static = "1"
@@ -18,10 +20,12 @@ liquid-derive = "0.26"
 path-absolutize = "3"
 pathdiff = "0.2"
 regex = { workspace = true }
+reqwest = { workspace = true }
 semver = "1"
 serde = { workspace = true }
 spin-common = { path = "../common" }
 spin-manifest = { path = "../manifest" }
+tar = "0.4"
 tempfile = { workspace = true }
 tokio = { workspace = true, features = ["fs", "process", "rt", "macros"] }
 toml = { workspace = true }

crates/templates/Cargo.toml

@@ -118,6 +118,7 @@ pub(crate) fn parse_manifest_toml(text: impl AsRef<str>) -> anyhow::Result<RawTe
 pub(crate) enum RawInstalledFrom {
     Git { git: String },
     File { dir: String },
+    RemoteTar { url: String },
 }
 
 pub(crate) fn parse_installed_from(text: impl AsRef<str>) -> Option<RawInstalledFrom> {

crates/templates/src/reader.rs

@@ -25,6 +25,12 @@ pub enum TemplateSource {
     /// Templates much be in a `/templates` directory under the specified
     /// root.
     File(PathBuf),
+    /// Install from a remote tarball.
+    ///
+    /// Templates should be in a `/templates` directory under the root of the tarball.
+    /// The implementation also allows for there to be a single root directory containing
+    /// the `templates` directory - this makes it compatible with GitHub release tarballs.
+    RemoteTar(Url),
 }
 
 /// Settings for installing templates from a Git repository.
@@ -72,6 +78,9 @@ impl TemplateSource {
                     None
                 }
             }
+            Self::RemoteTar(url) => Some(crate::reader::RawInstalledFrom::RemoteTar {
+                url: url.to_string(),
+            }),
         }
     }
 
@@ -96,13 +105,15 @@ impl TemplateSource {
         match self {
             Self::Git(git_source) => clone_local(git_source).await,
             Self::File(path) => check_local(path).await,
+            Self::RemoteTar(url) => download_untar_local(url).await,
         }
     }
 
     pub(crate) fn requires_copy(&self) -> bool {
         match self {
             Self::Git { .. } => true,
             Self::File(_) => false,
+            Self::RemoteTar(_) => true,
         }
     }
 }
@@ -192,6 +203,84 @@ async fn check_local(path: &Path) -> anyhow::Result<LocalTemplateSource> {
     }
 }
 
+/// Download a tarball to a temorary directory
+async fn download_untar_local(url: &Url) -> anyhow::Result<LocalTemplateSource> {
+    use bytes::buf::Buf;
+
+    let temp_dir = tempdir()?;
+    let path = temp_dir.path().to_owned();
+
+    let resp = reqwest::get(url.clone())
+        .await
+        .with_context(|| format!("Failed to download from {url}"))?;
+    let tar_content = resp
+        .bytes()
+        .await
+        .with_context(|| format!("Failed to download from {url}"))?;
+
+    let reader = flate2::read::GzDecoder::new(tar_content.reader());
+    let mut archive = tar::Archive::new(reader);
+    archive
+        .unpack(&path)
+        .context("Failed to unpack tar archive")?;
+
+    let templates_root = bypass_gh_added_root(path);
+
+    Ok(LocalTemplateSource {
+        root: templates_root,
+        _temp_dir: Some(temp_dir),
+    })
+}
+
+/// GitHub adds a prefix directory to release tarballs (e.g. spin-v3.0.0/...).
+/// We try to locate the repo root within the unpacked tarball.
+fn bypass_gh_added_root(unpack_dir: PathBuf) -> PathBuf {
+    // If the unpack dir directly contains a `templates` dir then we are done.
+    if has_templates_dir(&unpack_dir) {
+        return unpack_dir;
+    }
+
+    let Ok(dirs) = unpack_dir.read_dir() else {
+        // If we can't traverse the unpack directory then return it and
+        // let the top level try to make sense of it.
+        return unpack_dir;
+    };
+
+    // Is there a single directory at the root?  If not, we can't be in the GitHub situation:
+    // return the root of the unpacking. (The take(2) here is because we don't need to traverse
+    // the full list - we only care whether there is more than one.)
+    let dirs = dirs.filter_map(|de| de.ok()).take(2).collect::<Vec<_>>();
+    if dirs.len() != 1 {
+        return unpack_dir;
+    }
+
+    // If we get here, there is a single directory (dirs has a single element). Look in it to see if it's a plausible repo root.
+    let candidate_repo_root = dirs[0].path();
+    let Ok(mut candidate_repo_dirs) = candidate_repo_root.read_dir() else {
+        // Again, if it all goes awry, propose the base unpack directory.
+        return unpack_dir;
+    };
+    let has_templates_dir = candidate_repo_dirs.any(is_templates_dir);
+
+    if has_templates_dir {
+        candidate_repo_root
+    } else {
+        unpack_dir
+    }
+}
+
+fn has_templates_dir(path: &Path) -> bool {
+    let Ok(mut dirs) = path.read_dir() else {
+        return false;
+    };
+
+    dirs.any(is_templates_dir)
+}
+
+fn is_templates_dir(dir_entry: Result<std::fs::DirEntry, std::io::Error>) -> bool {
+    dir_entry.is_ok_and(|d| d.file_name() == TEMPLATE_SOURCE_DIR)
+}
+
 #[cfg(test)]
 mod test {
     use super::*;

crates/templates/src/source.rs

@@ -37,6 +37,7 @@ pub struct Template {
 enum InstalledFrom {
     Git(String),
     Directory(String),
+    RemoteTar(String),
     Unknown,
 }
 
@@ -254,6 +255,7 @@ impl Template {
         match &self.installed_from {
             InstalledFrom::Git(repo) => repo,
             InstalledFrom::Directory(path) => path,
+            InstalledFrom::RemoteTar(url) => url,
             InstalledFrom::Unknown => "",
         }
     }
@@ -625,6 +627,7 @@ fn read_install_record(layout: &TemplateLayout) -> InstalledFrom {
     match installed_from_text.and_then(parse_installed_from) {
         Some(RawInstalledFrom::Git { git }) => InstalledFrom::Git(git),
         Some(RawInstalledFrom::File { dir }) => InstalledFrom::Directory(dir),
+        Some(RawInstalledFrom::RemoteTar { url }) => InstalledFrom::RemoteTar(url),
         None => InstalledFrom::Unknown,
     }
 }

crates/templates/src/template.rs

@@ -15,6 +15,7 @@ use crate::build_info::*;
 
 const INSTALL_FROM_DIR_OPT: &str = "FROM_DIR";
 const INSTALL_FROM_GIT_OPT: &str = "FROM_GIT";
+const INSTALL_FROM_TAR_OPT: &str = "FROM_TAR";
 const UPGRADE_ONLY: &str = "GIT_URL";
 
 const DEFAULT_TEMPLATES_INSTALL_PROMPT: &str =
@@ -64,6 +65,7 @@ pub struct Install {
         long = "git",
         alias = "repo",
         conflicts_with = INSTALL_FROM_DIR_OPT,
+        conflicts_with = INSTALL_FROM_TAR_OPT,
     )]
     pub git: Option<String>,
 
@@ -76,9 +78,19 @@ pub struct Install {
         name = INSTALL_FROM_DIR_OPT,
         long = "dir",
         conflicts_with = INSTALL_FROM_GIT_OPT,
+        conflicts_with = INSTALL_FROM_TAR_OPT,
     )]
     pub dir: Option<PathBuf>,
 
+    /// URL to a tarball in .tar.gz format containing the template(s) to install.
+    #[clap(
+        name = INSTALL_FROM_TAR_OPT,
+        long = "tar",
+        conflicts_with = INSTALL_FROM_GIT_OPT,
+        conflicts_with = INSTALL_FROM_DIR_OPT,
+    )]
+    pub tar_url: Option<String>,
+
     /// If present, updates existing templates instead of skipping.
     #[clap(long = "upgrade", alias = "update")]
     pub update: bool,
@@ -119,16 +131,20 @@ impl Install {
     pub async fn run(self) -> Result<()> {
         let template_manager = TemplateManager::try_default()
             .context("Failed to construct template directory path")?;
-        let source = match (&self.git, &self.dir) {
-            (Some(git), None) => {
+        let source = match (&self.git, &self.dir, &self.tar_url) {
+            (Some(git), None, None) => {
                 let git_url = infer_github(git);
                 TemplateSource::try_from_git(git_url, &self.branch, SPIN_VERSION)?
             }
-            (None, Some(dir)) => {
+            (None, Some(dir), None) => {
                 let abs_dir = dir.absolutize().map(|d| d.to_path_buf());
                 TemplateSource::File(abs_dir.unwrap_or_else(|_| dir.clone()))
             }
-            _ => anyhow::bail!("Exactly one of `git` and `dir` sources must be specified"),
+            (None, None, Some(tar_url)) => {
+                let url = url::Url::parse(tar_url).context("Invalid URL for remote tar")?;
+                TemplateSource::RemoteTar(url)
+            }
+            _ => anyhow::bail!("Exactly one of `git`, `dir`, or `tar` must be specified"),
         };
 
         let reporter = ConsoleProgressReporter;
@@ -204,6 +220,7 @@ impl Upgrade {
                 git: self.git.clone(),
                 branch: self.branch.clone(),
                 dir: None,
+                tar_url: None,
                 update: true,
             };
 
@@ -620,6 +637,7 @@ async fn install_default_templates() -> anyhow::Result<()> {
         git: Some(DEFAULT_TEMPLATE_REPO.to_owned()),
         branch: None,
         dir: None,
+        tar_url: None,
         update: false,
     };
     install_cmd