feat: Add support for alternate download locations

brane-ctl/src/cli.rs

@@ -3,8 +3,8 @@
 
 use brane_cfg::proxy::ProxyProtocol;
 use brane_ctl::spec::{
-    API_DEFAULT_VERSION, DownloadServicesSubcommand, GenerateBackendSubcommand, GenerateCertsSubcommand, GenerateNodeSubcommand, InclusiveRange,
-    Pair, PolicyInputLanguage, ResolvableNodeKind, StartSubcommand, VersionFix,
+    API_DEFAULT_VERSION, GenerateBackendSubcommand, GenerateCertsSubcommand, GenerateNodeSubcommand, ImageGroup, InclusiveRange, Pair,
+    PolicyInputLanguage, ResolvableNodeKind, StartSubcommand, VersionFix,
 };
 use brane_tsk::docker::ClientVersion;
 use clap::{Parser, Subcommand};
@@ -15,6 +15,8 @@
 use specifications::package::Capability;
 use specifications::version::Version;
 
+const DEFAULT_DOCKER_HOST: &str = "/var/run/docker.sock";
+
 /***** ARGUMENTS *****/
 /// Defines the toplevel arguments for the `branectl` tool.
 #[derive(Debug, Parser)]
@@ -190,10 +192,7 @@
             global = true,
             help = "The processor architecture for which to download the images. Specify '$LOCAL' to use the architecture of the current machine."
         )]
-        arch:    Arch,
-        /// The version of the services to download.
-        #[clap(short, long, default_value=env!("CARGO_PKG_VERSION"), global=true, help="The version of the images to download from GitHub. You can specify 'latest' to download the latest version (but that might be incompatible with this CTL version)")]
-        version: Version,
+        arch:  Arch,
         /// Whether to overwrite existing images or not.
         #[clap(
             short = 'F',
@@ -202,11 +201,19 @@
             help = "If given, will overwrite services that are already there. Otherwise, these are not overwritten. Note that regardless, a \
                     download will still be performed."
         )]
-        force:   bool,
+        force: bool,
+
+        /// The path of the Docker socket.
+        #[clap(long, default_value = DEFAULT_DOCKER_HOST, env="DOCKER_HOST", help = "The path of the Docker socket to connect to.")]
+        docker_socket: PathBuf,
+        /// The client version to connect with.
+        #[clap(long, default_value=API_DEFAULT_VERSION.as_str(), env="DOCKER_API_VERSION", help="The client version to connect to the Docker instance with.")]
+        docker_client_version: ClientVersion,
 
         /// Whether to download the central or the worker VMs.
-        #[clap(subcommand)]
-        kind: DownloadServicesSubcommand,
+        /// TODO: Enhance docs
+        #[clap(help = "The collection of images")]
+        kind: String,
     },
 }
 

brane-ctl/src/download.rs

@@ -19,7 +19,8 @@ use std::io::Write as _;
 use std::path::{Component, Path, PathBuf};
 
 use brane_shr::fs::{DownloadSecurity, download_file_async, move_path_async, unarchive_async};
-use brane_tsk::docker::{Docker, DockerOptions, ImageSource, connect_local, ensure_image, save_image};
+use brane_shr::utilities::{ContainerImageSource, RepositoryRelease, create_dir_with_cachedirtag};
+use brane_tsk::docker::{ClientVersion, Docker, DockerOptions, ImageSource, connect_local, ensure_image, save_image};
 use console::{Style, style};
 use enum_debug::EnumDebug as _;
 use log::{debug, info, warn};
@@ -29,7 +30,7 @@ use specifications::version::Version;
 use tempfile::TempDir;
 
 pub use crate::errors::DownloadError as Error;
-use crate::spec::DownloadServicesSubcommand;
+use crate::spec::ImageGroup;
 
 
 /***** CONSTANTS *****/
@@ -153,6 +154,8 @@ async fn download_brane_services(address: impl AsRef<str>, path: impl AsRef<Path
 
 
 /***** LIBRARY *****/
+
+
 /// Downloads the service images to the local machine from the GitHub repo.
 ///
 /// # Arguments
@@ -169,126 +172,78 @@ pub async fn services(
     fix_dirs: bool,
     path: impl AsRef<Path>,
     arch: Arch,
-    version: Version,
     force: bool,
-    kind: DownloadServicesSubcommand,
+    docker_client: PathBuf,
+    docker_version: ClientVersion,
+    identifier: String,
 ) -> Result<(), Error> {
     let path: &Path = path.as_ref();
-    info!("Downloading {} service images...", kind.variant());
+    let image_source =
+        ContainerImageSource::from_identifier(&identifier, arch).map_err(|source| Error::InvalidDownloadIdentifier { identifier, source })?;
 
-    // Fix the missing directories, if any.
-    if !path.exists() {
-        // We are paralyzed if the user told us not to do anything
-        if !fix_dirs {
-            return Err(Error::DirNotFound { what: "output", path: path.into() });
-        }
+    info!("Downloading images");
 
-        // Else, generate the directory tree one-by-one. We place a CACHEDIR.TAG in the highest one we create.
-        let mut first: bool = true;
-        let mut stack: PathBuf = PathBuf::new();
-        for comp in path.components() {
-            match comp {
-                Component::RootDir => {
-                    stack = PathBuf::from("/");
-                    continue;
-                },
-                Component::Prefix(comp) => {
-                    stack = PathBuf::from(comp.as_os_str());
-                    continue;
-                },
-
-                Component::CurDir => continue,
-                Component::ParentDir => {
-                    stack.pop();
-                    continue;
-                },
-                Component::Normal(comp) => {
-                    stack.push(comp);
-                    if !stack.exists() {
-                        // Create the directory first
-                        fs::create_dir(&stack).map_err(|source| Error::DirCreateError { what: "output", path: stack.clone(), source })?;
-
-                        // Then create the CACHEDIR.TAG if we haven't already
-                        if first {
-                            let tag_path: PathBuf = stack.join("CACHEDIR.TAG");
-                            let mut handle: File =
-                                File::create(&tag_path).map_err(|source| Error::CachedirTagCreate { path: tag_path.clone(), source })?;
-                            handle.write(
-                                b"Signature: 8a477f597d28d172789f06886806bc55\n# This file is a cache directory tag created by BRANE's `branectl`.\n# For information about cache directory tags, see:\n#	    https://www.brynosaurus.com/cachedir/\n",
-                            ).map_err(|source| Error::CachedirTagWrite { path: tag_path, source })?;
-                            first = false;
-                        }
-                    }
-                    continue;
-                },
-            }
-        }
+    // We are paralyzed if the user told us not to do anything
+    if !path.exists() && !fix_dirs {
+        return Err(Error::DirNotFound { what: "output", path: path.into() });
     }
+
     if !path.is_dir() {
         return Err(Error::DirNotADir { what: "output", path: path.into() });
     }
 
-    // Now match on what we are downloading
-    match &kind {
-        DownloadServicesSubcommand::Central => {
-            // Resolve the address to use
-            let address: String = if version.is_latest() {
-                format!("https://github.com/braneframework/brane/releases/latest/download/instance-{}.tar.gz", arch.brane())
-            } else {
-                format!("https://github.com/braneframework/brane/releases/download/v{}/instance-{}.tar.gz", version, arch.brane())
-            };
-            debug!("Will download from: {}", address);
-
-            // Hand it over the shared code
-            download_brane_services(address, path, format!("instance-{}", arch.brane()), force).await?;
-        },
-
-        DownloadServicesSubcommand::Worker => {
-            // Resolve the address to use
-            let address: String = if version.is_latest() {
-                format!("https://github.com/braneframework/brane/releases/latest/download/worker-instance-{}.tar.gz", arch.brane())
-            } else {
-                format!("https://github.com/braneframework/brane/releases/download/v{}/worker-instance-{}.tar.gz", version, arch.brane())
-            };
-            debug!("Will download from: {}", address);
-
-            // Hand it over the shared code
-            download_brane_services(address, path, format!("worker-instance-{}", arch.brane()), force).await?;
-        },
+    create_dir_with_cachedirtag(path)?;
 
-        DownloadServicesSubcommand::Auxillary { socket, client_version } => {
+    match image_source {
+        ContainerImageSource::RegistryImage(registry_image) => {
             // Attempt to connect to the local Docker daemon.
-            let docker: Docker = connect_local(DockerOptions { socket: socket.clone(), version: *client_version })
+            let docker: Docker = connect_local(DockerOptions { socket: docker_client.clone(), version: docker_version })
                 .map_err(|source| Error::DockerConnectError { source })?;
 
-            // Download the pre-determined set of auxillary images
-            for (name, image) in AUXILLARY_DOCKER_IMAGES {
-                // We can skip it if it already exists
-                let image_path: PathBuf = path.join(format!("{name}.tar"));
-                if !force && image_path.exists() {
-                    debug!("Image '{}' already exists (skipping)", image_path.display());
-                    continue;
-                }
+            let name = registry_image.image.clone();
+            let image = registry_image.identifier();
 
+            // Download the pre-determined set of auxillary images
+            // We can skip it if it already exists
+            let image_path: PathBuf = path.join(format!("{name}.tar"));
+            if !force && image_path.exists() {
+                println!("Image '{}' already exists (skipping)", image_path.display());
+            } else {
                 // Make sure the image is pulled
-                println!("Downloading auxillary image {}...", style(image).bold().green());
-                ensure_image(&docker, Image::new(name, None::<&str>, None::<&str>), ImageSource::Registry(image.into()))
+                println!("Downloading auxillary image {}...", style(&image).bold().green());
+                ensure_image(&docker, Image::new(name.clone(), None::<&str>, None::<&str>), ImageSource::Registry(image.clone()))
                     .await
-                    .map_err(|source| Error::PullError { name: name.into(), image: image.into(), source })?;
+                    .map_err(|source| Error::PullError { name: name.clone(), image: image.clone(), source })?;
 
                 // Save the image to the correct path
-                println!("Exporting auxillary image {}...", style(name).bold().green());
-                save_image(&docker, Image::from(image), &image_path).await.map_err(|source| Error::SaveError {
-                    name: name.into(),
-                    image: image.into(),
+                println!("Exporting auxillary image {}...", style(&name).bold().green());
+                save_image(&docker, Image::from(image.clone()), &image_path).await.map_err(|source| Error::SaveError {
+                    name: name.clone(),
+                    image: image.clone(),
                     path: image_path,
                     source,
                 })?;
             }
         },
+        ContainerImageSource::RepositoryRelease(artifact) => {
+            let address = &artifact.url();
+
+            let archive_dir = address
+                .rsplit_once('/')
+                .ok_or_else(|| Error::InvalidDownloadUrl { url: address.clone() })?
+                .1
+                .strip_suffix(".tar.gz")
+                .ok_or_else(|| Error::InvalidDownloadUrl { url: address.clone() })?
+                .to_owned();
+
+            debug!("Will download from: {}", address);
+
+            // Hand it over the shared code
+            download_brane_services(address, path, archive_dir, force).await?;
+        },
     }
 
     // Done!
-    println!("Successfully downloaded {} services to {}", kind.variant().to_string().to_lowercase(), style(path.display()).bold().green());
+    // println!("Successfully downloaded {} services to {}", kind.variant().to_string().to_lowercase(), style(path.display()).bold().green());
     Ok(())
 }

brane-ctl/src/errors.rs

@@ -18,6 +18,7 @@ use std::process::{Command, ExitStatus};
 
 use brane_cfg::node::NodeKind;
 use brane_shr::formatters::Capitalizeable;
+use brane_shr::utilities::{ContainerImageSourceError, CreateDirWithCacheTagError};
 use brane_tsk::docker::ImageSource;
 use console::style;
 use enum_debug::EnumDebug as _;
@@ -32,22 +33,14 @@ use specifications::version::Version;
 /// Note: we box `brane_shr::fs::Error` to avoid the error enum growing too large (see `clippy::result_large_err`).
 #[derive(Debug, thiserror::Error)]
 pub enum DownloadError {
-    /// Failed to create a new CACHEDIR.TAG
-    #[error("Failed to create CACHEDIR.TAG file '{}'", path.display())]
-    CachedirTagCreate { path: PathBuf, source: std::io::Error },
-    /// Failed to write to a new CACHEDIR.TAG
-    #[error("Failed to write to CACHEDIR.TAG file '{}'", path.display())]
-    CachedirTagWrite { path: PathBuf, source: std::io::Error },
-
+    #[error("Could not create directory with CacheDir.Tag")]
+    CreateDirWithCacheDirTagError(#[from] CreateDirWithCacheTagError),
     /// The given directory does not exist.
     #[error("{} directory '{}' not found", what.capitalize(), path.display())]
     DirNotFound { what: &'static str, path: PathBuf },
     /// The given directory exists but is not a directory.
     #[error("{} directory '{}' exists but is not a directory", what.capitalize(), path.display())]
     DirNotADir { what: &'static str, path: PathBuf },
-    /// Could not create a new directory at the given location.
-    #[error("Failed to create {} directory '{}'", what, path.display())]
-    DirCreateError { what: &'static str, path: PathBuf, source: std::io::Error },
 
     /// Failed to create a temporary directory.
     #[error("Failed to create a temporary directory")]
@@ -77,6 +70,10 @@ pub enum DownloadError {
     /// Failed to save a pulled image.
     #[error("Failed to save image '{}' to '{}'", name, path.display())]
     SaveError { name: String, image: String, path: PathBuf, source: brane_tsk::docker::Error },
+    #[error("{url} is not a valid download url")]
+    InvalidDownloadUrl { url: String },
+    #[error("Invalid download identifier: {identifier}")]
+    InvalidDownloadIdentifier { identifier: String, source: ContainerImageSourceError },
 }
 
 

brane-ctl/src/main.rs

@@ -57,9 +57,9 @@ async fn main() {
     // Now match on the command
     match args.subcommand {
         CtlSubcommand::Download(subcommand) => match *subcommand {
-            DownloadSubcommand::Services { fix_dirs, path, arch, version, force, kind } => {
+            DownloadSubcommand::Services { fix_dirs, path, arch, force, docker_socket, docker_client_version, kind } => {
                 // Run the subcommand
-                if let Err(err) = download::services(fix_dirs, path, arch, version, force, kind).await {
+                if let Err(err) = download::services(fix_dirs, path, arch, force, docker_socket, docker_client_version, kind).await {
                     error!("{}", err.trace());
                     std::process::exit(1);
                 }

brane-ctl/src/spec.rs

@@ -19,7 +19,7 @@ use std::str::FromStr;
 
 use brane_cfg::node::NodeKind;
 use brane_tsk::docker::{ClientVersion, ImageSource};
-use clap::Subcommand;
+use clap::{Subcommand, ValueEnum};
 use enum_debug::EnumDebug;
 use specifications::address::Address;
 use specifications::version::Version;
@@ -300,30 +300,17 @@ pub struct LogsOpts {
     pub compose_verbose: bool,
 }
 
-
-
-/// A bit awkward here, but defines the subcommand for downloading service images from the repo.
-#[derive(Debug, EnumDebug, Subcommand)]
-pub enum DownloadServicesSubcommand {
+#[derive(Clone, Debug, EnumDebug, ValueEnum)]
+pub enum ImageGroup {
     /// Download the services for a central node.
-    #[clap(name = "central", about = "Downloads the central node services (brane-api, brane-drv, brane-plr, brane-prx)")]
+    #[clap(name = "central")]
     Central,
     /// Download the services for a worker node.
-    #[clap(name = "worker", about = "Downloads the worker node services (brane-reg, brane-job, brane-prx)")]
+    #[clap(name = "worker")]
     Worker,
     /// Download the auxillary services for the central node.
-    #[clap(
-        name = "auxillary",
-        about = "Downloads the auxillary services for the central node. Note that most of these are actually downloaded using Docker."
-    )]
-    Auxillary {
-        /// The path of the Docker socket.
-        #[clap(short, long, default_value = "/var/run/docker.sock", help = "The path of the Docker socket to connect to.")]
-        socket: PathBuf,
-        /// The client version to connect with.
-        #[clap(short, long, default_value=API_DEFAULT_VERSION.as_str(), help="The client version to connect to the Docker instance with.")]
-        client_version: ClientVersion,
-    },
+    #[clap(name = "auxillary")]
+    Auxillary,
 }
 
 /// A bit awkward here, but defines the generate subcommand for the node file. This basically defines the possible kinds of nodes to generate.