repository: Add new "blob" feature

This is sort of similar to splitstream, except its just a blob of
data with references, rather than something intended to be spliced
together from objects.

The short term usecases are ostree commit data, and object mappings,
but it is flexible enough to be useful for other things too.

Signed-off-by: Alexander Larsson <[email protected]>

Author: alexlarsson

crates/composefs/src/blob.rs

@@ -0,0 +1,97 @@
+/* Implementation of the Split Stream file format
+ *
+ * See doc/splitstream.md
+ */
+
+use std::{
+    io::{Read, Write},
+    sync::Arc,
+};
+
+use anyhow::{bail, Result};
+
+use crate::{fsverity::FsVerityHashValue, repository::Repository};
+
+const BLOB_MAGIC_V1: u64 = 0xAFE138C18C463EF3;
+
+#[derive(Debug)]
+pub struct BlobWriter<ObjectID: FsVerityHashValue> {
+    repo: Arc<Repository<ObjectID>>,
+    pub refs: Vec<ObjectID>,
+    content: Vec<u8>,
+}
+
+impl<ObjectID: FsVerityHashValue> BlobWriter<ObjectID> {
+    pub fn new(repo: &Arc<Repository<ObjectID>>) -> Self {
+        Self {
+            repo: Arc::clone(repo),
+            content: vec![],
+            refs: vec![],
+        }
+    }
+
+    pub fn add_reference(&mut self, reference: &ObjectID) {
+        self.refs.push(reference.clone())
+    }
+
+    pub fn done(&self) -> Result<ObjectID> {
+        let mut res = Vec::<u8>::new();
+        res.extend_from_slice(&u64::to_le_bytes(BLOB_MAGIC_V1));
+        res.extend_from_slice(&u64::to_le_bytes(self.refs.len() as u64));
+        for obj_id in self.refs.iter() {
+            res.extend_from_slice(obj_id.as_bytes());
+        }
+        res.extend_from_slice(&self.content);
+
+        self.repo.ensure_object(&res)
+    }
+}
+
+impl<ObjectID: FsVerityHashValue> Write for BlobWriter<ObjectID> {
+    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        self.content.extend_from_slice(buf);
+        Ok(buf.len())
+    }
+
+    fn flush(&mut self) -> std::io::Result<()> {
+        Ok(())
+    }
+}
+
+#[derive(Debug)]
+pub struct BlobReader<R: Read, ObjectID: FsVerityHashValue> {
+    reader: R,
+    pub refs: Vec<ObjectID>,
+}
+
+impl<R: Read, ObjectID: FsVerityHashValue> BlobReader<R, ObjectID> {
+    pub fn new(mut reader: R) -> Result<Self> {
+        let magic = {
+            let mut buf = [0u8; 8];
+            reader.read_exact(&mut buf)?;
+            u64::from_le_bytes(buf)
+        };
+        if magic != BLOB_MAGIC_V1 {
+            bail!("Invalida blob header magic value");
+        }
+
+        let n_refs = {
+            let mut buf = [0u8; 8];
+            reader.read_exact(&mut buf)?;
+            u64::from_le_bytes(buf)
+        } as usize;
+
+        let mut refs = Vec::with_capacity(n_refs);
+        for _ in 0..n_refs {
+            refs.push(ObjectID::read_from_io(&mut reader)?);
+        }
+
+        Ok(Self { reader, refs })
+    }
+}
+
+impl<F: Read, ObjectID: FsVerityHashValue> Read for BlobReader<F, ObjectID> {
+    fn read(&mut self, data: &mut [u8]) -> std::io::Result<usize> {
+        self.reader.read(data)
+    }
+}

crates/composefs/src/lib.rs

@@ -1,3 +1,4 @@
+pub mod blob;
 pub mod dumpfile;
 pub mod dumpfile_parse;
 pub mod erofs;

crates/composefs/src/repository.rs

@@ -13,14 +13,15 @@ use once_cell::sync::OnceCell;
 use rand::{distr::Alphanumeric, Rng};
 use rustix::{
     fs::{
-        fdatasync, flock, linkat, mkdirat, open, openat, readlinkat, symlinkat, AtFlags, Dir,
-        FileType, FlockOperation, Mode, OFlags, CWD,
+        fdatasync, flock, linkat, mkdirat, open, openat, readlinkat, renameat, symlinkat, AtFlags,
+        Dir, FileType, FlockOperation, Mode, OFlags, CWD,
     },
     io::{Errno, Result as ErrnoResult},
 };
 use sha2::{Digest, Sha256};
 
 use crate::{
+    blob::{BlobReader, BlobWriter},
     fsverity::{
         compute_verity, enable_verity, ensure_verity_equal, measure_verity, FsVerityHashValue,
     },
@@ -362,6 +363,59 @@ impl<ObjectID: FsVerityHashValue> Repository<ObjectID> {
         Ok(())
     }
 
+    pub fn has_named_blob(&self, name: &str) -> bool {
+        let blob_path = format!("blobs/refs/{}", name);
+
+        match readlinkat(&self.repository, &blob_path, []) {
+            Ok(_) => true,
+            Err(_) => false,
+        }
+    }
+
+    /// Creates a Blobriter for writing a blob.
+    /// You should write the data to the returned object and then pass it to .store_blob() to
+    /// store the result.
+    pub fn create_blob(self: &Arc<Self>) -> BlobWriter<ObjectID> {
+        BlobWriter::new(self)
+    }
+
+    pub fn write_blob(&self, writer: BlobWriter<ObjectID>, name: Option<&str>) -> Result<ObjectID> {
+        let object_id = writer.done()?;
+
+        let object_path = Self::format_object_path(&object_id);
+        let blob_path = format!("blobs/{}", object_id.to_hex());
+
+        self.ensure_symlink(&blob_path, &object_path)?;
+
+        if let Some(reference) = name {
+            let ref_path = format!("blobs/refs/{reference}");
+            self.symlink(&ref_path, &blob_path)?;
+        }
+
+        Ok(object_id)
+    }
+
+    pub fn name_blob(&self, object_id: &ObjectID, name: &str) -> Result<()> {
+        let blob_path = format!("blobs/{}", object_id.to_hex());
+        let reference_path = format!("blobs/refs/{name}");
+        self.symlink(&reference_path, &blob_path)?;
+        Ok(())
+    }
+
+    pub fn open_blob(&self, name: &str) -> Result<BlobReader<File, ObjectID>> {
+        let fd = self
+            .openat(&format!("blobs/{name}"), OFlags::RDONLY)
+            .with_context(|| format!("Opening ref 'blobs/{name}'"))?;
+
+        if !name.contains("/") {
+            // A name with no slashes in it is taken to be a sha256 fs-verity digest
+            ensure_verity_equal(&fd, &ObjectID::from_hex(name)?)?;
+        }
+
+        let file = File::from(fd);
+        BlobReader::new(file)
+    }
+
     /// this function is not safe for untrusted users
     pub fn write_image(&self, name: Option<&str>, data: &[u8]) -> Result<ObjectID> {
         let object_id = self.ensure_object(data)?;
@@ -593,6 +647,17 @@ impl<ObjectID: FsVerityHashValue> Repository<ObjectID> {
             })?;
         }
 
+        for object in self.gc_category("blobs")? {
+            println!("{object:?} lives as a blob");
+            objects.insert(object.clone());
+
+            let blob = self.open_blob(&object.to_hex())?;
+            for reference in blob.refs {
+                println!("   with {reference:?}");
+                objects.insert(reference.clone());
+            }
+        }
+
         for first_byte in 0x0..=0xff {
             let dirfd = match self.openat(
                 &format!("objects/{first_byte:02x}"),

doc/blobs.md

@@ -0,0 +1,35 @@
+# Blobs
+
+Blobs are a way to reference generic data objects other than images in
+the repository. The blobs are a way to reference data objects, which
+otherwise would be garbage collected.
+
+Additionally, the blob file format allows the blob to references a
+list of other objects, that will also be kept alive. Other than this
+the blob format itself is wihtout structure, and it is up to each user
+to define this.
+
+Similar to images, blobs can also be given names, which makes it
+easy to find them.
+
+## Example uses
+
+Blobs are used by composefs-ostree to store native ostree
+representations of ostree commits that can be converted to
+images, but also used as a source for previous data when
+updating an ostree commit.
+
+## File format
+
+The file format consists of a header, plus one data block for the
+remainder of the file.
+
+### Header
+
+The file starts with a magic number, 0xAFE138C18C463EF3 as a u64 in
+little endian. After that comes a single u64 le integer which
+specified the number of references other objects. Directl after that,
+the references objects are stored by fsverity hash value (32/64
+bytes).
+
+The remainder of the file is of unspecified format.

doc/repository.md

@@ -40,6 +40,10 @@ composefs
 │   ├── 4e67eaccd9fd[...] -> ../objects/4e/67eaccd9fd[...]
 │   └── refs
 │       └── some/name -> ../../images/4e67eaccd9fd[...]
+├── blobs
+│   ├── bfa4c6a073a1[...] -> ../objects/bf/a4c6a073a1[...]
+│   └── refs
+│       └── some/name -> ../../images/4e67eaccd9fd[...]
 └── streams
     ├── 502b126bca0c[...] -> ../objects/50/2b126bca0c[...]
     └── refs
@@ -86,10 +90,17 @@ repository as a splitstream, the resulting filename in this directory will have
 no relation to the original content.  You can, however, store a reference for
 it.
 
-## `{images,streams}/refs/`
+## `blobs/`
+
+This is where [blobs](blob.md) are stored.  As for the images,
+this is a bunch of 256bit symlinks which are symlinks to data in the object
+storage.
+
+
+## `{images,streams,blobs}/refs/`
 
-This is where we record which images and streams are currently "requested" by
-some external user.  When importing a tar file, in addition to creating the
+This is where we record which images, blobs and streams are currently "requested"
+by some external user.  When importing e.g. a tar file, in addition to creating the
 file in the objects database and the toplevel symlink in the `streams/`
 directory, we also assign it a name which is chosen by the software which is
 performing the import.
@@ -126,9 +137,9 @@ prevent users from corrupting the layout of the repository.  The reason for the
 acl is that read-only operations on the repository should be performed
 directly on the repository and not via some central agent.
 
-## Referring to images and streams
+## Referring to images, blobs and streams
 
-Operations that are performed on images or streams (mount, cat, etc.) name the
+Operations that are performed on images, blobs or streams (mount, cat, etc.) name the
 stream in one of two ways:
 
  - via the user-chosen name such as `refs/1000/flatpak/some_id`
@@ -137,7 +148,7 @@ stream in one of two ways:
 ie: the name must either start with the string `refs/`, or must be a 64bit
 character hexidecimal string.
 
-In both cases, the name is a path relative to the `images/` or `streams/`
+In both cases, the name is a path relative to the `images/`, `blobs/` or `streams/`
 directory and this path contains a symlink (either direct or indirect) to the
 underlying file in `objects/`.