diff --git a/Cargo.lock b/Cargo.lock index 5242d2147..ff3fec34a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -254,6 +254,7 @@ dependencies = [ "libsystemd", "linkme", "nom", + "ocidir", "openssl", "ostree-ext", "regex", @@ -265,6 +266,7 @@ dependencies = [ "serde_yaml", "similar-asserts", "static_assertions", + "tar", "tempfile", "thiserror 2.0.17", "tini", diff --git a/crates/lib/Cargo.toml b/crates/lib/Cargo.toml index ab03da40d..fb700ddd6 100644 --- a/crates/lib/Cargo.toml +++ b/crates/lib/Cargo.toml @@ -62,9 +62,11 @@ liboverdrop = "0.1.0" libsystemd = "0.7" linkme = "0.3" nom = "8.0.0" +ocidir = "0.6.0" schemars = { version = "1.0.4", features = ["chrono04"] } serde_ignored = "0.1.10" serde_yaml = "0.9.34" +tar = "0.4.43" tini = "1.3.0" uuid = { version = "1.8.0", features = ["v4"] } uapi-version = "0.4.0" diff --git a/crates/lib/src/bootc_composefs/export.rs b/crates/lib/src/bootc_composefs/export.rs new file mode 100644 index 000000000..cef2c3ec9 --- /dev/null +++ b/crates/lib/src/bootc_composefs/export.rs @@ -0,0 +1,220 @@ +use std::{fs::File, os::fd::AsRawFd}; + +use anyhow::{Context, Result}; +use cap_std_ext::cap_std::{ambient_authority, fs::Dir}; +use composefs::splitstream::SplitStreamData; +use ocidir::{oci_spec::image::Platform, OciDir}; +use ostree_ext::container::skopeo; +use ostree_ext::{container::Transport, oci_spec::image::ImageConfiguration}; +use tar::EntryType; + +use crate::image::get_imgrefs_for_copy; +use crate::{ + bootc_composefs::{ + status::{get_composefs_status, get_imginfo}, + update::str_to_sha256digest, + }, + store::{BootedComposefs, Storage}, +}; + +/// Exports a composefs repository to a container image in containers-storage: +pub async fn export_repo_to_image( + storage: &Storage, + booted_cfs: &BootedComposefs, + source: Option<&str>, + target: Option<&str>, +) -> Result<()> { + let host = get_composefs_status(storage, booted_cfs).await?; + + let (source, dest_imgref) = get_imgrefs_for_copy(&host, source, target).await?; + + let mut depl_verity = None; + + for depl in host + .status + .booted + .iter() + .chain(host.status.staged.iter()) + .chain(host.status.rollback.iter()) + .chain(host.status.other_deployments.iter()) + { + let img = &depl.image.as_ref().unwrap().image; + + // Not checking transport here as we'll be pulling from the repo anyway + // So, image name is all we need + if img.image == source.name { + depl_verity = Some(depl.require_composefs()?.verity.clone()); + break; + } + } + + let depl_verity = depl_verity.ok_or_else(|| anyhow::anyhow!("Image {source} not found"))?; + + let imginfo = get_imginfo(storage, &depl_verity, None).await?; + + let config_name = &imginfo.manifest.config().digest().digest(); + let config_name = str_to_sha256digest(config_name)?; + + let var_tmp = + Dir::open_ambient_dir("/var/tmp", ambient_authority()).context("Opening /var/tmp")?; + + let tmpdir = cap_std_ext::cap_tempfile::tempdir_in(&var_tmp)?; + let oci_dir = OciDir::ensure(tmpdir.try_clone()?).context("Opening OCI")?; + + let mut config_stream = booted_cfs + .repo + .open_stream(&hex::encode(config_name), None) + .context("Opening config stream")?; + + let config = ImageConfiguration::from_reader(&mut config_stream)?; + + // We can't guarantee that we'll get the same tar stream as the container image + // So we create new config and manifest + let mut new_config = config.clone(); + if let Some(history) = new_config.history_mut() { + history.clear(); + } + new_config.rootfs_mut().diff_ids_mut().clear(); + + let mut new_manifest = imginfo.manifest.clone(); + new_manifest.layers_mut().clear(); + + let total_layers = config.rootfs().diff_ids().len(); + + for (idx, old_diff_id) in config.rootfs().diff_ids().iter().enumerate() { + let layer_sha256 = str_to_sha256digest(old_diff_id)?; + let layer_verity = config_stream.lookup(&layer_sha256)?; + + let mut layer_stream = booted_cfs + .repo + .open_stream(&hex::encode(layer_sha256), Some(layer_verity))?; + + let mut layer_writer = oci_dir.create_layer(None)?; + layer_writer.follow_symlinks(false); + + let mut got_zero_block = false; + + loop { + let mut buf = [0u8; 512]; + + if !layer_stream + .read_inline_exact(&mut buf) + .context("Reading into buffer")? + { + break; + } + + let all_zeroes = buf.iter().all(|x| *x == 0); + + // EOF for tar + if all_zeroes && got_zero_block { + break; + } else if all_zeroes { + got_zero_block = true; + continue; + } + + got_zero_block = false; + + let header = tar::Header::from_byte_slice(&buf); + + let size = header.entry_size()?; + + match layer_stream.read_exact(size as usize, ((size + 511) & !511) as usize)? { + SplitStreamData::External(obj_id) => match header.entry_type() { + EntryType::Regular | EntryType::Continuous => { + let file = File::from(booted_cfs.repo.open_object(&obj_id)?); + + layer_writer + .append(&header, file) + .context("Failed to write external entry")?; + } + + _ => anyhow::bail!("Unsupported external-chunked entry {header:?} {obj_id:?}"), + }, + + SplitStreamData::Inline(content) => match header.entry_type() { + EntryType::Directory => { + layer_writer.append(&header, std::io::empty())?; + } + + // We do not care what the content is as we're re-archiving it anyway + _ => { + layer_writer + .append(&header, &*content) + .context("Failed to write inline entry")?; + } + }, + }; + } + + layer_writer.finish()?; + + let layer = layer_writer + .into_inner() + .context("Getting inner layer writer")? + .complete() + .context("Writing layer to disk")?; + + tracing::debug!( + "Wrote layer: {layer_sha} #{layer_num}/{total_layers}", + layer_sha = layer.uncompressed_sha256_as_digest(), + layer_num = idx + 1, + ); + + let previous_annotations = imginfo + .manifest + .layers() + .get(idx) + .and_then(|l| l.annotations().as_ref()) + .cloned(); + + let history = imginfo.config.history().as_ref(); + let history_entry = history.and_then(|v| v.get(idx)); + let previous_description = history_entry + .clone() + .and_then(|h| h.comment().as_deref()) + .unwrap_or_default(); + + let previous_created = history_entry + .and_then(|h| h.created().as_deref()) + .and_then(bootc_utils::try_deserialize_timestamp) + .unwrap_or_default(); + + oci_dir.push_layer_full( + &mut new_manifest, + &mut new_config, + layer, + previous_annotations, + previous_description, + previous_created, + ); + } + + let descriptor = oci_dir.write_config(new_config).context("Writing config")?; + + new_manifest.set_config(descriptor); + oci_dir + .insert_manifest(new_manifest, None, Platform::default()) + .context("Writing manifest")?; + + // Pass the temporary oci directory as the current working directory for the skopeo process + let tempoci = ostree_ext::container::ImageReference { + transport: Transport::OciDir, + name: format!("/proc/self/fd/{}", tmpdir.as_raw_fd()), + }; + + skopeo::copy( + &tempoci, + &dest_imgref, + None, + Some(( + std::sync::Arc::new(tmpdir.try_clone()?.into()), + tmpdir.as_raw_fd(), + )), + true, + ) + .await?; + + Ok(()) +} diff --git a/crates/lib/src/bootc_composefs/mod.rs b/crates/lib/src/bootc_composefs/mod.rs index ddc88c516..d3dda0f5f 100644 --- a/crates/lib/src/bootc_composefs/mod.rs +++ b/crates/lib/src/bootc_composefs/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod boot; pub(crate) mod delete; pub(crate) mod digest; +pub(crate) mod export; pub(crate) mod finalize; pub(crate) mod gc; pub(crate) mod repo; diff --git a/crates/lib/src/bootc_composefs/repo.rs b/crates/lib/src/bootc_composefs/repo.rs index c3f478169..f97f76f44 100644 --- a/crates/lib/src/bootc_composefs/repo.rs +++ b/crates/lib/src/bootc_composefs/repo.rs @@ -61,7 +61,7 @@ pub(crate) fn get_imgref(transport: &str, image: &str) -> String { let img = image.strip_prefix(":").unwrap_or(&image); let transport = transport.strip_suffix(":").unwrap_or(&transport); - if transport == "registry" { + if transport == "registry" || transport == "docker://" { format!("docker://{img}") } else if transport == "docker-daemon" { format!("docker-daemon:{img}") diff --git a/crates/lib/src/bootc_composefs/status.rs b/crates/lib/src/bootc_composefs/status.rs index 579c506d9..97c53c12e 100644 --- a/crates/lib/src/bootc_composefs/status.rs +++ b/crates/lib/src/bootc_composefs/status.rs @@ -198,7 +198,10 @@ pub(crate) async fn get_container_manifest_and_config( let config = containers_image_proxy::ImageProxyConfig::default(); let proxy = containers_image_proxy::ImageProxy::new_with_config(config).await?; - let img = proxy.open_image(&imgref).await.context("Opening image")?; + let img = proxy + .open_image(&imgref) + .await + .with_context(|| format!("Opening image {imgref}"))?; let (_, manifest) = proxy.fetch_manifest(&img).await?; let (mut reader, driver) = proxy.get_descriptor(&img, manifest.config()).await?; @@ -238,7 +241,7 @@ pub(crate) fn get_bootloader() -> Result { pub(crate) async fn get_imginfo( storage: &Storage, deployment_id: &str, - imgref: &ImageReference, + imgref: Option<&ImageReference>, ) -> Result { let imginfo_fname = format!("{deployment_id}.imginfo"); @@ -251,6 +254,8 @@ pub(crate) async fn get_imginfo( .context("Failed to open file")?; let Some(img_conf) = &mut img_conf else { + let imgref = imgref.ok_or_else(|| anyhow::anyhow!("No imgref or imginfo file found"))?; + let container_details = get_container_manifest_and_config(&get_imgref(&imgref.transport, &imgref.image)) .await?; @@ -291,7 +296,7 @@ async fn boot_entry_from_composefs_deployment( let ostree_img_ref = OstreeImageReference::from_str(&img_name_from_config)?; let img_ref = ImageReference::from(ostree_img_ref); - let img_conf = get_imginfo(storage, &verity, &img_ref).await?; + let img_conf = get_imginfo(storage, &verity, Some(&img_ref)).await?; let image_digest = img_conf.manifest.config().digest().to_string(); let architecture = img_conf.config.architecture().to_string(); diff --git a/crates/lib/src/bootc_composefs/update.rs b/crates/lib/src/bootc_composefs/update.rs index 84afdb416..d910e7350 100644 --- a/crates/lib/src/bootc_composefs/update.rs +++ b/crates/lib/src/bootc_composefs/update.rs @@ -411,7 +411,7 @@ pub(crate) async fn upgrade_composefs( if opts.check { let current_manifest = - get_imginfo(storage, &*composefs.cmdline.digest, booted_imgref).await?; + get_imginfo(storage, &*composefs.cmdline.digest, Some(booted_imgref)).await?; let diff = ManifestDiff::new(¤t_manifest.manifest, &img_config.manifest); diff.print(); return Ok(()); diff --git a/crates/lib/src/cli.rs b/crates/lib/src/cli.rs index c4705419e..bf7cad787 100644 --- a/crates/lib/src/cli.rs +++ b/crates/lib/src/cli.rs @@ -43,14 +43,15 @@ use crate::bootc_composefs::{ update::upgrade_composefs, }; use crate::deploy::{MergeState, RequiredHostSpec}; -use crate::lints; use crate::podstorage::set_additional_image_store; use crate::progress_jsonl::{ProgressWriter, RawProgressFd}; use crate::spec::Host; use crate::spec::ImageReference; +use crate::status::get_host; use crate::store::{BootedOstree, Storage}; use crate::store::{BootedStorage, BootedStorageKind}; use crate::utils::sigpolicy_from_opt; +use crate::{bootc_composefs, lints}; /// Shared progress options #[derive(Debug, Parser, PartialEq, Eq)] @@ -1586,8 +1587,33 @@ async fn run_from_opt(opt: Opt) -> Result<()> { list_type, list_format, } => crate::image::list_entrypoint(list_type, list_format).await, + ImageOpts::CopyToStorage { source, target } => { - crate::image::push_entrypoint(source.as_deref(), target.as_deref()).await + // We get "host" here to avoid deadlock in the ostree path + let host = get_host().await?; + + let storage = get_storage().await?; + + match storage.kind()? { + BootedStorageKind::Ostree(..) => { + crate::image::push_entrypoint( + &storage, + &host, + source.as_deref(), + target.as_deref(), + ) + .await + } + BootedStorageKind::Composefs(booted) => { + bootc_composefs::export::export_repo_to_image( + &storage, + &booted, + source.as_deref(), + target.as_deref(), + ) + .await + } + } } ImageOpts::SetUnified => crate::image::set_unified_entrypoint().await, ImageOpts::PullFromDefaultStorage { image } => { diff --git a/crates/lib/src/image.rs b/crates/lib/src/image.rs index b2e3882cf..4aaff294a 100644 --- a/crates/lib/src/image.rs +++ b/crates/lib/src/image.rs @@ -15,11 +15,13 @@ use crate::{ boundimage::query_bound_images, cli::{ImageListFormat, ImageListType}, podstorage::CStorage, + spec::Host, + store::Storage, utils::async_task_with_spinner, }; /// The name of the image we push to containers-storage if nothing is specified. -const IMAGE_DEFAULT: &str = "localhost/bootc"; +pub(crate) const IMAGE_DEFAULT: &str = "localhost/bootc"; /// Check if an image exists in the default containers-storage (podman storage). /// @@ -139,43 +141,66 @@ pub(crate) async fn list_entrypoint( Ok(()) } -/// Implementation of `bootc image push-to-storage`. -#[context("Pushing image")] -pub(crate) async fn push_entrypoint(source: Option<&str>, target: Option<&str>) -> Result<()> { +/// Returns the source and target ImageReference +/// If the source isn't specified, we use booted image +/// If the target isn't specified, we push to containers-storage with our default image +pub(crate) async fn get_imgrefs_for_copy( + host: &Host, + source: Option<&str>, + target: Option<&str>, +) -> Result<(ImageReference, ImageReference)> { // Initialize floating c_storage early - needed for container operations crate::podstorage::ensure_floating_c_storage_initialized(); - let transport = Transport::ContainerStorage; - let sysroot = crate::cli::get_storage().await?; - let ostree = sysroot.get_ostree()?; - let repo = &ostree.repo(); - // If the target isn't specified, push to containers-storage + our default image - let target = if let Some(target) = target { - ImageReference { - transport, + let dest_imgref = match target { + Some(target) => ostree_ext::container::ImageReference { + transport: Transport::ContainerStorage, name: target.to_owned(), - } - } else { - ImageReference { + }, + None => ostree_ext::container::ImageReference { transport: Transport::ContainerStorage, - name: IMAGE_DEFAULT.to_string(), - } + name: IMAGE_DEFAULT.into(), + }, }; // If the source isn't specified, we use the booted image - let source = if let Some(source) = source { - ImageReference::try_from(source).context("Parsing source image")? - } else { - let status = crate::status::get_status_require_booted(&ostree)?; - // SAFETY: We know it's booted - let booted = status.2.status.booted.unwrap(); - let booted_image = booted.image.unwrap().image; - ImageReference { - transport: Transport::try_from(booted_image.transport.as_str()).unwrap(), - name: booted_image.image, + let src_imgref = match source { + Some(source) => ostree_ext::container::ImageReference::try_from(source) + .context("Parsing source image")?, + + None => { + let booted = host + .status + .booted + .as_ref() + .ok_or_else(|| anyhow::anyhow!("Booted deployment not found"))?; + + let booted_image = &booted.image.as_ref().unwrap().image; + + ImageReference { + transport: Transport::try_from(booted_image.transport.as_str()).unwrap(), + name: booted_image.image.clone(), + } } }; + + return Ok((src_imgref, dest_imgref)); +} + +/// Implementation of `bootc image push-to-storage`. +#[context("Pushing image")] +pub(crate) async fn push_entrypoint( + storage: &Storage, + host: &Host, + source: Option<&str>, + target: Option<&str>, +) -> Result<()> { + let (source, target) = get_imgrefs_for_copy(host, source, target).await?; + + let ostree = storage.get_ostree()?; + let repo = &ostree.repo(); + let mut opts = ostree_ext::container::store::ExportToOCIOpts::default(); opts.progress_to_stdout = true; println!("Copying local image {source} to {target} ..."); diff --git a/crates/lib/src/status.rs b/crates/lib/src/status.rs index e682c9923..20bf11e6c 100644 --- a/crates/lib/src/status.rs +++ b/crates/lib/src/status.rs @@ -430,7 +430,7 @@ pub(crate) fn get_status( Ok((deployments, host)) } -async fn get_host() -> Result { +pub(crate) async fn get_host() -> Result { let env = crate::store::Environment::detect()?; if env.needs_mount_namespace() { crate::cli::prepare_for_write()?; diff --git a/crates/ostree-ext/src/container/mod.rs b/crates/ostree-ext/src/container/mod.rs index 5c252478a..77d8e1717 100644 --- a/crates/ostree-ext/src/container/mod.rs +++ b/crates/ostree-ext/src/container/mod.rs @@ -483,7 +483,7 @@ mod encapsulate; pub use encapsulate::*; mod unencapsulate; pub use unencapsulate::*; -mod skopeo; +pub mod skopeo; pub mod store; mod update_detachedmeta; pub use update_detachedmeta::*; diff --git a/crates/ostree-ext/src/container/skopeo.rs b/crates/ostree-ext/src/container/skopeo.rs index 7ac462509..eceaf0935 100644 --- a/crates/ostree-ext/src/container/skopeo.rs +++ b/crates/ostree-ext/src/container/skopeo.rs @@ -64,7 +64,7 @@ pub(crate) fn spawn(mut cmd: Command) -> Result { /// Use skopeo to copy a container image. #[context("Skopeo copy")] -pub(crate) async fn copy( +pub async fn copy( src: &ImageReference, dest: &ImageReference, authfile: Option<&Path>,