Preparatory splitstream format changes for ostree support

crates/cfsctl/src/main.rs

@@ -48,7 +48,7 @@ pub struct App {
 enum OciCommand {
     /// Stores a tar file as a splitstream in the repository.
     ImportLayer {
-        sha256: String,
+        digest: String,
         name: Option<String>,
     },
     /// Lists the contents of a tar stream
@@ -105,7 +105,7 @@ enum Command {
     Transaction,
     /// Reconstitutes a split stream and writes it to stdout
     Cat {
-        /// the name of the stream to cat, either a sha256 digest or prefixed with 'ref/'
+        /// the name of the stream to cat, either a content identifier or prefixed with 'ref/'
         name: String,
     },
     /// Perform garbage collection
@@ -122,7 +122,7 @@ enum Command {
     },
     /// Mounts a composefs, possibly enforcing fsverity of the image
     Mount {
-        /// the name of the image to mount, either a sha256 digest or prefixed with 'ref/'
+        /// the name of the image to mount, either an fs-verity hash or prefixed with 'ref/'
         name: String,
         /// the mountpoint
         mountpoint: String,
@@ -194,18 +194,18 @@ async fn main() -> Result<()> {
             }
         }
         Command::Cat { name } => {
-            repo.merge_splitstream(&name, None, &mut std::io::stdout())?;
+            repo.merge_splitstream(&name, None, None, &mut std::io::stdout())?;
         }
         Command::ImportImage { reference } => {
             let image_id = repo.import_image(&reference, &mut std::io::stdin())?;
             println!("{}", image_id.to_id());
         }
         #[cfg(feature = "oci")]
         Command::Oci { cmd: oci_cmd } => match oci_cmd {
-            OciCommand::ImportLayer { name, sha256 } => {
+            OciCommand::ImportLayer { name, digest } => {
                 let object_id = composefs_oci::import_layer(
                     &Arc::new(repo),
-                    &composefs::util::parse_sha256(sha256)?,
+                    &digest,
                     name.as_deref(),
                     &mut std::io::stdin(),
                 )?;
@@ -253,20 +253,20 @@ async fn main() -> Result<()> {
                 println!("{}", image_id.to_id());
             }
             OciCommand::Pull { ref image, name } => {
-                let (sha256, verity) =
+                let (digest, verity) =
                     composefs_oci::pull(&Arc::new(repo), image, name.as_deref(), None).await?;
 
-                println!("sha256 {}", hex::encode(sha256));
+                println!("config {digest}");
                 println!("verity {}", verity.to_hex());
             }
             OciCommand::Seal {
                 ref config_name,
                 ref config_verity,
             } => {
                 let verity = verity_opt(config_verity)?;
-                let (sha256, verity) =
+                let (digest, verity) =
                     composefs_oci::seal(&Arc::new(repo), config_name, verity.as_ref())?;
-                println!("sha256 {}", hex::encode(sha256));
+                println!("config {digest}");
                 println!("verity {}", verity.to_id());
             }
             OciCommand::Mount {
@@ -367,8 +367,8 @@ async fn main() -> Result<()> {
         }
         #[cfg(feature = "http")]
         Command::Fetch { url, name } => {
-            let (sha256, verity) = composefs_http::download(&url, &name, Arc::new(repo)).await?;
-            println!("sha256 {}", hex::encode(sha256));
+            let (digest, verity) = composefs_http::download(&url, &name, Arc::new(repo)).await?;
+            println!("content {digest}");
             println!("verity {}", verity.to_hex());
         }
     }

crates/composefs-http/src/lib.rs

@@ -7,7 +7,6 @@
 use std::{
     collections::{HashMap, HashSet},
     fs::File,
-    io::Read,
     sync::Arc,
 };
 
@@ -19,10 +18,7 @@ use sha2::{Digest, Sha256};
 use tokio::task::JoinSet;
 
 use composefs::{
-    fsverity::FsVerityHashValue,
-    repository::Repository,
-    splitstream::{DigestMapEntry, SplitStreamReader},
-    util::Sha256Digest,
+    fsverity::FsVerityHashValue, repository::Repository, splitstream::SplitStreamReader,
 };
 
 struct Downloader<ObjectID: FsVerityHashValue> {
@@ -66,17 +62,11 @@ impl<ObjectID: FsVerityHashValue> Downloader<ObjectID> {
         }
     }
 
-    fn open_splitstream(&self, id: &ObjectID) -> Result<SplitStreamReader<File, ObjectID>> {
-        SplitStreamReader::new(File::from(self.repo.open_object(id)?))
+    fn open_splitstream(&self, id: &ObjectID) -> Result<SplitStreamReader<ObjectID>> {
+        SplitStreamReader::new(File::from(self.repo.open_object(id)?), None)
     }
 
-    fn read_object(&self, id: &ObjectID) -> Result<Vec<u8>> {
-        let mut data = vec![];
-        File::from(self.repo.open_object(id)?).read_to_end(&mut data)?;
-        Ok(data)
-    }
-
-    async fn ensure_stream(self: &Arc<Self>, name: &str) -> Result<(Sha256Digest, ObjectID)> {
+    async fn ensure_stream(self: &Arc<Self>, name: &str) -> Result<(String, ObjectID)> {
         let progress = ProgressBar::new(2); // the first object gets "ensured" twice
         progress.set_style(
             ProgressStyle::with_template(
@@ -113,8 +103,8 @@ impl<ObjectID: FsVerityHashValue> Downloader<ObjectID> {
 
             // this part is fast: it only touches the header
             let mut reader = self.open_splitstream(&id)?;
-            for DigestMapEntry { verity, body } in &reader.refs.map {
-                match splitstreams.insert(verity.clone(), Some(*body)) {
+            for (body, verity) in reader.iter_named_refs() {
+                match splitstreams.insert(verity.clone(), Some(body.to_string())) {
                     // This is the (normal) case if we encounter a splitstream we didn't see yet...
                     None => {
                         splitstreams_todo.push(verity.clone());
@@ -125,7 +115,7 @@ impl<ObjectID: FsVerityHashValue> Downloader<ObjectID> {
                     // verify the SHA-256 content hashes later (after we get all the objects) so we
                     // need to make sure that all referents of this stream agree on what that is.
                     Some(Some(previous)) => {
-                        if previous != *body {
+                        if previous != body {
                             bail!(
                                 "Splitstream with verity {verity:?} has different body hashes {} and {}",
                                 hex::encode(previous),
@@ -208,8 +198,8 @@ impl<ObjectID: FsVerityHashValue> Downloader<ObjectID> {
         for (id, expected_checksum) in splitstreams {
             let mut reader = self.open_splitstream(&id)?;
             let mut context = Sha256::new();
-            reader.cat(&mut context, |id| self.read_object(id))?;
-            let measured_checksum: Sha256Digest = context.finalize().into();
+            reader.cat(&self.repo, &mut context)?;
+            let measured_checksum = format!("sha256:{}", hex::encode(context.finalize()));
 
             if let Some(expected) = expected_checksum {
                 if measured_checksum != expected {
@@ -265,7 +255,7 @@ pub async fn download<ObjectID: FsVerityHashValue>(
     url: &str,
     name: &str,
     repo: Arc<Repository<ObjectID>>,
-) -> Result<(Sha256Digest, ObjectID)> {
+) -> Result<(String, ObjectID)> {
     let downloader = Arc::new(Downloader {
         client: Client::new(),
         repo,

crates/composefs-oci/src/image.rs

@@ -11,14 +11,15 @@
 use std::{ffi::OsStr, os::unix::ffi::OsStrExt, rc::Rc};
 
 use anyhow::{ensure, Context, Result};
-use oci_spec::image::ImageConfiguration;
+use sha2::{Digest, Sha256};
 
 use composefs::{
     fsverity::FsVerityHashValue,
     repository::Repository,
     tree::{Directory, FileSystem, Inode, Leaf},
 };
 
+use crate::skopeo::TAR_LAYER_CONTENT_TYPE;
 use crate::tar::{TarEntry, TarItem};
 
 /// Processes a single tar entry and adds it to the filesystem.
@@ -84,21 +85,38 @@ pub fn process_entry<ObjectID: FsVerityHashValue>(
 
 /// Creates a filesystem from the given OCI container.  No special transformations are performed to
 /// make the filesystem bootable.
+///
+/// If `config_verity` is given it is used to get the OCI config splitstream by its fs-verity ID
+/// and the entire process is substantially faster.  If it is not given, the config and layers will
+/// be hashed to ensure that they match their claimed blob IDs.
 pub fn create_filesystem<ObjectID: FsVerityHashValue>(
     repo: &Repository<ObjectID>,
     config_name: &str,
     config_verity: Option<&ObjectID>,
 ) -> Result<FileSystem<ObjectID>> {
     let mut filesystem = FileSystem::default();
 
-    let mut config_stream = repo.open_stream(config_name, config_verity)?;
-    let config = ImageConfiguration::from_reader(&mut config_stream)?;
+    let (config, map) = crate::open_config(repo, config_name, config_verity)?;
 
     for diff_id in config.rootfs().diff_ids() {
-        let layer_sha256 = super::sha256_from_digest(diff_id)?;
-        let layer_verity = config_stream.lookup(&layer_sha256)?;
+        let layer_verity = map
+            .get(diff_id.as_str())
+            .context("OCI config splitstream missing named ref to layer {diff_id}")?;
+
+        if config_verity.is_none() {
+            // We don't have any proof that the named references in the config splitstream are
+            // trustworthy. We have no choice but to perform expensive validation of the layer
+            // stream.
+            let mut layer_stream =
+                repo.open_stream("", Some(layer_verity), Some(TAR_LAYER_CONTENT_TYPE))?;
+            let mut context = Sha256::new();
+            layer_stream.cat(repo, &mut context)?;
+            let content_hash = format!("sha256:{}", hex::encode(context.finalize()));
+            ensure!(content_hash == *diff_id, "Layer has incorrect checksum");
+        }
 
-        let mut layer_stream = repo.open_stream(&hex::encode(layer_sha256), Some(layer_verity))?;
+        let mut layer_stream =
+            repo.open_stream("", Some(layer_verity), Some(TAR_LAYER_CONTENT_TYPE))?;
         while let Some(entry) = crate::tar::get_entry(&mut layer_stream)? {
             process_entry(&mut filesystem, entry)?;
         }