Implement bootc image copy-to-storage for composefs backend

crates/lib/Cargo.toml

@@ -62,9 +62,11 @@ liboverdrop = "0.1.0"
 libsystemd = "0.7"
 linkme = "0.3"
 nom = "8.0.0"
+ocidir = "0.6.0"
 schemars = { version = "1.0.4", features = ["chrono04"] }
 serde_ignored = "0.1.10"
 serde_yaml = "0.9.34"
+tar = "0.4.43"
 tini = "1.3.0"
 uuid = { version = "1.8.0", features = ["v4"] }
 uapi-version = "0.4.0"

crates/lib/src/bootc_composefs/export.rs

@@ -0,0 +1,220 @@
+use std::{fs::File, os::fd::AsRawFd};
+
+use anyhow::{Context, Result};
+use cap_std_ext::cap_std::{ambient_authority, fs::Dir};
+use composefs::splitstream::SplitStreamData;
+use ocidir::{oci_spec::image::Platform, OciDir};
+use ostree_ext::container::skopeo;
+use ostree_ext::{container::Transport, oci_spec::image::ImageConfiguration};
+use tar::EntryType;
+
+use crate::image::get_imgrefs_for_copy;
+use crate::{
+    bootc_composefs::{
+        status::{get_composefs_status, get_imginfo},
+        update::str_to_sha256digest,
+    },
+    store::{BootedComposefs, Storage},
+};
+
+/// Exports a composefs repository to a container image in containers-storage:
+pub async fn export_repo_to_image(
+    storage: &Storage,
+    booted_cfs: &BootedComposefs,
+    source: Option<&str>,
+    target: Option<&str>,
+) -> Result<()> {
+    let host = get_composefs_status(storage, booted_cfs).await?;
+
+    let (source, dest_imgref) = get_imgrefs_for_copy(&host, source, target).await?;
+
+    let mut depl_verity = None;
+
+    for depl in host
+        .status
+        .booted
+        .iter()
+        .chain(host.status.staged.iter())
+        .chain(host.status.rollback.iter())
+        .chain(host.status.other_deployments.iter())
+    {
+        let img = &depl.image.as_ref().unwrap().image;
+
+        // Not checking transport here as we'll be pulling from the repo anyway
+        // So, image name is all we need
+        if img.image == source.name {
+            depl_verity = Some(depl.require_composefs()?.verity.clone());
+            break;
+        }
+    }
+
+    let depl_verity = depl_verity.ok_or_else(|| anyhow::anyhow!("Image {source} not found"))?;
+
+    let imginfo = get_imginfo(storage, &depl_verity, None).await?;
+
+    let config_name = &imginfo.manifest.config().digest().digest();
+    let config_name = str_to_sha256digest(config_name)?;
+
+    let var_tmp =
+        Dir::open_ambient_dir("/var/tmp", ambient_authority()).context("Opening /var/tmp")?;
+
+    let tmpdir = cap_std_ext::cap_tempfile::tempdir_in(&var_tmp)?;
+    let oci_dir = OciDir::ensure(tmpdir.try_clone()?).context("Opening OCI")?;
+
+    let mut config_stream = booted_cfs
+        .repo
+        .open_stream(&hex::encode(config_name), None)
+        .context("Opening config stream")?;
+
+    let config = ImageConfiguration::from_reader(&mut config_stream)?;
+
+    // We can't guarantee that we'll get the same tar stream as the container image
+    // So we create new config and manifest
+    let mut new_config = config.clone();
+    if let Some(history) = new_config.history_mut() {
+        history.clear();
+    }
+    new_config.rootfs_mut().diff_ids_mut().clear();
+
+    let mut new_manifest = imginfo.manifest.clone();
+    new_manifest.layers_mut().clear();
+
+    let total_layers = config.rootfs().diff_ids().len();
+
+    for (idx, old_diff_id) in config.rootfs().diff_ids().iter().enumerate() {
+        let layer_sha256 = str_to_sha256digest(old_diff_id)?;
+        let layer_verity = config_stream.lookup(&layer_sha256)?;
+
+        let mut layer_stream = booted_cfs
+            .repo
+            .open_stream(&hex::encode(layer_sha256), Some(layer_verity))?;
+
+        let mut layer_writer = oci_dir.create_layer(None)?;
+        layer_writer.follow_symlinks(false);
+
+        let mut got_zero_block = false;
+
+        loop {
+            let mut buf = [0u8; 512];
+
+            if !layer_stream
+                .read_inline_exact(&mut buf)
+                .context("Reading into buffer")?
+            {
+                break;
+            }
+
+            let all_zeroes = buf.iter().all(|x| *x == 0);
+
+            // EOF for tar
+            if all_zeroes && got_zero_block {
+                break;
+            } else if all_zeroes {
+                got_zero_block = true;
+                continue;
+            }
+
+            got_zero_block = false;
+
+            let header = tar::Header::from_byte_slice(&buf);
+
+            let size = header.entry_size()?;
+
+            match layer_stream.read_exact(size as usize, ((size + 511) & !511) as usize)? {
+                SplitStreamData::External(obj_id) => match header.entry_type() {
+                    EntryType::Regular | EntryType::Continuous => {
+                        let file = File::from(booted_cfs.repo.open_object(&obj_id)?);
+
+                        layer_writer
+                            .append(&header, file)
+                            .context("Failed to write external entry")?;
+                    }
+
+                    _ => anyhow::bail!("Unsupported external-chunked entry {header:?} {obj_id:?}"),
+                },
+
+                SplitStreamData::Inline(content) => match header.entry_type() {
+                    EntryType::Directory => {
+                        layer_writer.append(&header, std::io::empty())?;
+                    }
+
+                    // We do not care what the content is as we're re-archiving it anyway
+                    _ => {
+                        layer_writer
+                            .append(&header, &*content)
+                            .context("Failed to write inline entry")?;
+                    }
+                },
+            };
+        }
+
+        layer_writer.finish()?;
+
+        let layer = layer_writer
+            .into_inner()
+            .context("Getting inner layer writer")?
+            .complete()
+            .context("Writing layer to disk")?;
+
+        tracing::debug!(
+            "Wrote layer: {layer_sha} #{layer_num}/{total_layers}",
+            layer_sha = layer.uncompressed_sha256_as_digest(),
+            layer_num = idx + 1,
+        );
+
+        let previous_annotations = imginfo
+            .manifest
+            .layers()
+            .get(idx)
+            .and_then(|l| l.annotations().as_ref())
+            .cloned();
+
+        let history = imginfo.config.history().as_ref();
+        let history_entry = history.and_then(|v| v.get(idx));
+        let previous_description = history_entry
+            .clone()
+            .and_then(|h| h.comment().as_deref())
+            .unwrap_or_default();
+
+        let previous_created = history_entry
+            .and_then(|h| h.created().as_deref())
+            .and_then(bootc_utils::try_deserialize_timestamp)
+            .unwrap_or_default();
+
+        oci_dir.push_layer_full(
+            &mut new_manifest,
+            &mut new_config,
+            layer,
+            previous_annotations,
+            previous_description,
+            previous_created,
+        );
+    }
+
+    let descriptor = oci_dir.write_config(new_config).context("Writing config")?;
+
+    new_manifest.set_config(descriptor);
+    oci_dir
+        .insert_manifest(new_manifest, None, Platform::default())
+        .context("Writing manifest")?;
+
+    // Pass the temporary oci directory as the current working directory for the skopeo process
+    let tempoci = ostree_ext::container::ImageReference {
+        transport: Transport::OciDir,
+        name: format!("/proc/self/fd/{}", tmpdir.as_raw_fd()),
+    };
+
+    skopeo::copy(
+        &tempoci,
+        &dest_imgref,
+        None,
+        Some((
+            std::sync::Arc::new(tmpdir.try_clone()?.into()),
+            tmpdir.as_raw_fd(),
+        )),
+        true,
+    )
+    .await?;
+
+    Ok(())
+}

crates/lib/src/bootc_composefs/mod.rs

@@ -1,6 +1,7 @@
 pub(crate) mod boot;
 pub(crate) mod delete;
 pub(crate) mod digest;
+pub(crate) mod export;
 pub(crate) mod finalize;
 pub(crate) mod gc;
 pub(crate) mod repo;

crates/lib/src/bootc_composefs/repo.rs

@@ -61,7 +61,7 @@ pub(crate) fn get_imgref(transport: &str, image: &str) -> String {
     let img = image.strip_prefix(":").unwrap_or(&image);
     let transport = transport.strip_suffix(":").unwrap_or(&transport);
 
-    if transport == "registry" {
+    if transport == "registry" || transport == "docker://" {
         format!("docker://{img}")
     } else if transport == "docker-daemon" {
         format!("docker-daemon:{img}")

crates/lib/src/bootc_composefs/status.rs

@@ -198,7 +198,10 @@ pub(crate) async fn get_container_manifest_and_config(
     let config = containers_image_proxy::ImageProxyConfig::default();
     let proxy = containers_image_proxy::ImageProxy::new_with_config(config).await?;
 
-    let img = proxy.open_image(&imgref).await.context("Opening image")?;
+    let img = proxy
+        .open_image(&imgref)
+        .await
+        .with_context(|| format!("Opening image {imgref}"))?;
 
     let (_, manifest) = proxy.fetch_manifest(&img).await?;
     let (mut reader, driver) = proxy.get_descriptor(&img, manifest.config()).await?;
@@ -238,7 +241,7 @@ pub(crate) fn get_bootloader() -> Result<Bootloader> {
 pub(crate) async fn get_imginfo(
     storage: &Storage,
     deployment_id: &str,
-    imgref: &ImageReference,
+    imgref: Option<&ImageReference>,
 ) -> Result<ImgConfigManifest> {
     let imginfo_fname = format!("{deployment_id}.imginfo");
 
@@ -251,6 +254,8 @@ pub(crate) async fn get_imginfo(
         .context("Failed to open file")?;
 
     let Some(img_conf) = &mut img_conf else {
+        let imgref = imgref.ok_or_else(|| anyhow::anyhow!("No imgref or imginfo file found"))?;
+
         let container_details =
             get_container_manifest_and_config(&get_imgref(&imgref.transport, &imgref.image))
                 .await?;
@@ -291,7 +296,7 @@ async fn boot_entry_from_composefs_deployment(
             let ostree_img_ref = OstreeImageReference::from_str(&img_name_from_config)?;
             let img_ref = ImageReference::from(ostree_img_ref);
 
-            let img_conf = get_imginfo(storage, &verity, &img_ref).await?;
+            let img_conf = get_imginfo(storage, &verity, Some(&img_ref)).await?;
 
             let image_digest = img_conf.manifest.config().digest().to_string();
             let architecture = img_conf.config.architecture().to_string();

crates/lib/src/bootc_composefs/update.rs

@@ -411,7 +411,7 @@ pub(crate) async fn upgrade_composefs(
 
     if opts.check {
         let current_manifest =
-            get_imginfo(storage, &*composefs.cmdline.digest, booted_imgref).await?;
+            get_imginfo(storage, &*composefs.cmdline.digest, Some(booted_imgref)).await?;
         let diff = ManifestDiff::new(&current_manifest.manifest, &img_config.manifest);
         diff.print();
         return Ok(());

crates/lib/src/cli.rs

@@ -43,14 +43,15 @@ use crate::bootc_composefs::{
     update::upgrade_composefs,
 };
 use crate::deploy::{MergeState, RequiredHostSpec};
-use crate::lints;
 use crate::podstorage::set_additional_image_store;
 use crate::progress_jsonl::{ProgressWriter, RawProgressFd};
 use crate::spec::Host;
 use crate::spec::ImageReference;
+use crate::status::get_host;
 use crate::store::{BootedOstree, Storage};
 use crate::store::{BootedStorage, BootedStorageKind};
 use crate::utils::sigpolicy_from_opt;
+use crate::{bootc_composefs, lints};
 
 /// Shared progress options
 #[derive(Debug, Parser, PartialEq, Eq)]
@@ -1586,8 +1587,33 @@ async fn run_from_opt(opt: Opt) -> Result<()> {
                 list_type,
                 list_format,
             } => crate::image::list_entrypoint(list_type, list_format).await,
+
             ImageOpts::CopyToStorage { source, target } => {
-                crate::image::push_entrypoint(source.as_deref(), target.as_deref()).await
+                // We get "host" here to avoid deadlock in the ostree path
+                let host = get_host().await?;
+
+                let storage = get_storage().await?;
+
+                match storage.kind()? {
+                    BootedStorageKind::Ostree(..) => {
+                        crate::image::push_entrypoint(
+                            &storage,
+                            &host,
+                            source.as_deref(),
+                            target.as_deref(),
+                        )
+                        .await
+                    }
+                    BootedStorageKind::Composefs(booted) => {
+                        bootc_composefs::export::export_repo_to_image(
+                            &storage,
+                            &booted,
+                            source.as_deref(),
+                            target.as_deref(),
+                        )
+                        .await
+                    }
+                }
             }
             ImageOpts::SetUnified => crate::image::set_unified_entrypoint().await,
             ImageOpts::PullFromDefaultStorage { image } => {