Skip to content

Preparatory splitstream format changes for ostree support #185

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Preparatory splitstream format changes for ostree support #185

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion crates/cfsctl/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@ async fn main() -> Result<()> {
}
}
Command::Cat { name } => {
repo.merge_splitstream(&name, None, &mut std::io::stdout())?;
repo.merge_splitstream(&name, None, None, &mut std::io::stdout())?;
}
Command::ImportImage { reference } => {
let image_id = repo.import_image(&reference, &mut std::io::stdin())?;
Expand Down
8 changes: 3 additions & 5 deletions crates/composefs-http/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,7 @@ use sha2::{Digest, Sha256};
use tokio::task::JoinSet;

use composefs::{
fsverity::FsVerityHashValue,
repository::Repository,
splitstream::{DigestMapEntry, SplitStreamReader},
fsverity::FsVerityHashValue, repository::Repository, splitstream::SplitStreamReader,
util::Sha256Digest,
};

Expand Down Expand Up @@ -61,7 +59,7 @@ impl<ObjectID: FsVerityHashValue> Downloader<ObjectID> {
}

fn open_splitstream(&self, id: &ObjectID) -> Result<SplitStreamReader<File, ObjectID>> {
SplitStreamReader::new(File::from(self.repo.open_object(id)?))
SplitStreamReader::new(File::from(self.repo.open_object(id)?), None)
}

fn read_object(&self, id: &ObjectID) -> Result<Vec<u8>> {
Expand Down Expand Up @@ -107,7 +105,7 @@ impl<ObjectID: FsVerityHashValue> Downloader<ObjectID> {

// this part is fast: it only touches the header
let mut reader = self.open_splitstream(&id)?;
for DigestMapEntry { verity, body } in &reader.refs.map {
for (body, verity) in reader.iter_mappings() {
match splitstreams.insert(verity.clone(), Some(*body)) {
// This is the (normal) case if we encounter a splitstream we didn't see yet...
None => {
Expand Down
10 changes: 8 additions & 2 deletions crates/composefs-oci/src/image.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ use composefs::{
tree::{Directory, FileSystem, Inode, Leaf},
};

use crate::skopeo::{OCI_CONFIG_CONTENT_TYPE, TAR_LAYER_CONTENT_TYPE};
use crate::tar::{TarEntry, TarItem};

pub fn process_entry<ObjectID: FsVerityHashValue>(
Expand Down Expand Up @@ -74,14 +75,19 @@ pub fn create_filesystem<ObjectID: FsVerityHashValue>(
) -> Result<FileSystem<ObjectID>> {
let mut filesystem = FileSystem::default();

let mut config_stream = repo.open_stream(config_name, config_verity)?;
let mut config_stream =
repo.open_stream(config_name, config_verity, Some(OCI_CONFIG_CONTENT_TYPE))?;
let config = ImageConfiguration::from_reader(&mut config_stream)?;

for diff_id in config.rootfs().diff_ids() {
let layer_sha256 = super::sha256_from_digest(diff_id)?;
let layer_verity = config_stream.lookup(&layer_sha256)?;

let mut layer_stream = repo.open_stream(&hex::encode(layer_sha256), Some(layer_verity))?;
let mut layer_stream = repo.open_stream(
&hex::encode(layer_sha256),
Some(layer_verity),
Some(TAR_LAYER_CONTENT_TYPE),
)?;
while let Some(entry) = crate::tar::get_entry(&mut layer_stream)? {
process_entry(&mut filesystem, entry)?;
}
Expand Down
21 changes: 14 additions & 7 deletions crates/composefs-oci/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ use composefs::{
util::{parse_sha256, Sha256Digest},
};

use crate::skopeo::{OCI_CONFIG_CONTENT_TYPE, TAR_LAYER_CONTENT_TYPE};
use crate::tar::get_entry;

type ContentAndVerity<ObjectID> = (Sha256Digest, ObjectID);
Expand All @@ -40,14 +41,19 @@ pub fn import_layer<ObjectID: FsVerityHashValue>(
name: Option<&str>,
tar_stream: &mut impl Read,
) -> Result<ObjectID> {
repo.ensure_stream(sha256, |writer| tar::split(tar_stream, writer), name)
repo.ensure_stream(
sha256,
TAR_LAYER_CONTENT_TYPE,
|writer| tar::split(tar_stream, writer),
name,
)
}

pub fn ls_layer<ObjectID: FsVerityHashValue>(
repo: &Repository<ObjectID>,
name: &str,
) -> Result<()> {
let mut split_stream = repo.open_stream(name, None)?;
let mut split_stream = repo.open_stream(name, None, Some(TAR_LAYER_CONTENT_TYPE))?;

while let Some(entry) = get_entry(&mut split_stream)? {
println!("{entry}");
Expand Down Expand Up @@ -83,9 +89,9 @@ pub fn open_config<ObjectID: FsVerityHashValue>(
.with_context(|| format!("Object {name} is unknown to us"))?
}
};
let mut stream = repo.open_stream(name, Some(id))?;
let mut stream = repo.open_stream(name, Some(id), Some(OCI_CONFIG_CONTENT_TYPE))?;
let config = ImageConfiguration::from_reader(&mut stream)?;
Ok((config, stream.refs))
Ok((config, stream.get_mappings()))
}

fn hash(bytes: &[u8]) -> Sha256Digest {
Expand All @@ -106,7 +112,7 @@ pub fn open_config_shallow<ObjectID: FsVerityHashValue>(
// we need to manually check the content digest
let expected_hash = parse_sha256(name)
.context("Containers must be referred to by sha256 if verity is missing")?;
let mut stream = repo.open_stream(name, None)?;
let mut stream = repo.open_stream(name, None, Some(OCI_CONFIG_CONTENT_TYPE))?;
let mut raw_config = vec![];
stream.read_to_end(&mut raw_config)?;
ensure!(hash(&raw_config) == expected_hash, "Data integrity issue");
Expand All @@ -123,7 +129,8 @@ pub fn write_config<ObjectID: FsVerityHashValue>(
let json = config.to_string()?;
let json_bytes = json.as_bytes();
let sha256 = hash(json_bytes);
let mut stream = repo.create_stream(Some(sha256), Some(refs));
let mut stream = repo.create_stream(OCI_CONFIG_CONTENT_TYPE, Some(sha256));
stream.add_sha256_mappings(refs);
stream.write_inline(json_bytes);
let id = repo.write_stream(stream, None)?;
Ok((sha256, id))
Expand Down Expand Up @@ -201,7 +208,7 @@ mod test {
let id = import_layer(&repo, &layer_id, Some("name"), &mut layer.as_slice()).unwrap();

let mut dump = String::new();
let mut split_stream = repo.open_stream("refs/name", Some(&id)).unwrap();
let mut split_stream = repo.open_stream("refs/name", Some(&id), None).unwrap();
while let Some(entry) = tar::get_entry(&mut split_stream).unwrap() {
writeln!(dump, "{entry}").unwrap();
}
Expand Down
22 changes: 13 additions & 9 deletions crates/composefs-oci/src/skopeo.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,14 @@ use oci_spec::image::{Descriptor, ImageConfiguration, ImageManifest, MediaType};
use rustix::process::geteuid;
use tokio::{io::AsyncReadExt, sync::Semaphore};

use composefs::{
fsverity::FsVerityHashValue, repository::Repository, splitstream::DigestMap, util::Sha256Digest,
};
use composefs::{fsverity::FsVerityHashValue, repository::Repository, util::Sha256Digest};

use crate::{sha256_from_descriptor, sha256_from_digest, tar::split_async, ContentAndVerity};

// These are randomly generated UUID-like content types
pub const TAR_LAYER_CONTENT_TYPE: u64 = 0x2a037edfcae1ffea;
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you add a comment for where these came from? I'm guessing random? If so just add a comment // Random unique ID ?

That said I wonder if it wouldn't be nicer to store (variable length) strings for this in the format? Maybe it could go all the way to literally suggested to be the mediaType from OCI (if applicable)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes. These are random.

But, I'd rather avoid having variable length things in the header. That makes parsing it much more tricky.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We could make it a real uuid tho

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm fine keeping as u64 too.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I added some comments here

pub const OCI_CONFIG_CONTENT_TYPE: u64 = 0x44218c839727a80b;

struct ImageOp<ObjectID: FsVerityHashValue> {
repo: Arc<Repository<ObjectID>>,
proxy: ImageProxy,
Expand Down Expand Up @@ -95,7 +97,9 @@ impl<ObjectID: FsVerityHashValue> ImageOp<ObjectID> {
self.progress
.println(format!("Fetching layer {}", hex::encode(layer_sha256)))?;

let mut splitstream = self.repo.create_stream(Some(layer_sha256), None);
let mut splitstream = self
.repo
.create_stream(TAR_LAYER_CONTENT_TYPE, Some(layer_sha256));
match descriptor.media_type() {
MediaType::ImageLayer => {
split_async(progress, &mut splitstream).await?;
Expand Down Expand Up @@ -172,15 +176,15 @@ impl<ObjectID: FsVerityHashValue> ImageOp<ObjectID> {
entries.push((layer_sha256, future));
}

let mut splitstream = self
.repo
.create_stream(OCI_CONFIG_CONTENT_TYPE, Some(config_sha256));

// Collect the results.
let mut config_maps = DigestMap::new();
for (layer_sha256, future) in entries {
config_maps.insert(&layer_sha256, &future.await??);
splitstream.add_sha256_mapping(&layer_sha256, &future.await??);
}

let mut splitstream = self
.repo
.create_stream(Some(config_sha256), Some(config_maps));
splitstream.write_inline(&raw_config);
let config_id = self.repo.write_stream(splitstream, None)?;

Expand Down
47 changes: 32 additions & 15 deletions crates/composefs-oci/src/tar.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -281,7 +281,10 @@ pub fn get_entry<R: Read, ObjectID: FsVerityHashValue>(
}

#[cfg(test)]

mod tests {
use crate::TAR_LAYER_CONTENT_TYPE;

use super::*;
use composefs::{
fsverity::Sha256HashValue, generic_tree::LeafContent, repository::Repository,
Expand Down Expand Up @@ -338,13 +341,15 @@ mod tests {
fn read_all_via_splitstream(tar_data: Vec<u8>) -> Result<Vec<TarEntry<Sha256HashValue>>> {
let mut tar_cursor = Cursor::new(tar_data);
let repo = create_test_repository()?;
let mut writer = repo.create_stream(None, None);
let mut writer = repo.create_stream(TAR_LAYER_CONTENT_TYPE, None);

split(&mut tar_cursor, &mut writer)?;
let object_id = writer.done()?;

let mut reader: SplitStreamReader<std::fs::File, Sha256HashValue> =
SplitStreamReader::new(repo.open_object(&object_id)?.into())?;
let mut reader: SplitStreamReader<std::fs::File, Sha256HashValue> = SplitStreamReader::new(
repo.open_object(&object_id)?.into(),
Some(TAR_LAYER_CONTENT_TYPE),
)?;

let mut entries = Vec::new();
while let Some(entry) = get_entry(&mut reader)? {
Expand All @@ -363,13 +368,16 @@ mod tests {

let mut tar_cursor = Cursor::new(tar_data);
let repo = create_test_repository().unwrap();
let mut writer = repo.create_stream(None, None);
let mut writer = repo.create_stream(TAR_LAYER_CONTENT_TYPE, None);

split(&mut tar_cursor, &mut writer).unwrap();
let object_id = writer.done().unwrap();

let mut reader: SplitStreamReader<std::fs::File, Sha256HashValue> =
SplitStreamReader::new(repo.open_object(&object_id).unwrap().into()).unwrap();
let mut reader: SplitStreamReader<std::fs::File, Sha256HashValue> = SplitStreamReader::new(
repo.open_object(&object_id).unwrap().into(),
Some(TAR_LAYER_CONTENT_TYPE),
)
.unwrap();
assert!(get_entry(&mut reader).unwrap().is_none());
}

Expand All @@ -389,13 +397,16 @@ mod tests {

let mut tar_cursor = Cursor::new(tar_data);
let repo = create_test_repository().unwrap();
let mut writer = repo.create_stream(None, None);
let mut writer = repo.create_stream(TAR_LAYER_CONTENT_TYPE, None);

split(&mut tar_cursor, &mut writer).unwrap();
let object_id = writer.done().unwrap();

let mut reader: SplitStreamReader<std::fs::File, Sha256HashValue> =
SplitStreamReader::new(repo.open_object(&object_id).unwrap().into()).unwrap();
let mut reader: SplitStreamReader<std::fs::File, Sha256HashValue> = SplitStreamReader::new(
repo.open_object(&object_id).unwrap().into(),
Some(TAR_LAYER_CONTENT_TYPE),
)
.unwrap();

// Should have exactly one entry
let entry = get_entry(&mut reader)
Expand Down Expand Up @@ -444,13 +455,16 @@ mod tests {

let mut tar_cursor = Cursor::new(tar_data);
let repo = create_test_repository().unwrap();
let mut writer = repo.create_stream(None, None);
let mut writer = repo.create_stream(TAR_LAYER_CONTENT_TYPE, None);

split(&mut tar_cursor, &mut writer).unwrap();
let object_id = writer.done().unwrap();

let mut reader: SplitStreamReader<std::fs::File, Sha256HashValue> =
SplitStreamReader::new(repo.open_object(&object_id).unwrap().into()).unwrap();
let mut reader: SplitStreamReader<std::fs::File, Sha256HashValue> = SplitStreamReader::new(
repo.open_object(&object_id).unwrap().into(),
Some(TAR_LAYER_CONTENT_TYPE),
)
.unwrap();
let mut entries = Vec::new();

while let Some(entry) = get_entry(&mut reader).unwrap() {
Expand Down Expand Up @@ -508,13 +522,16 @@ mod tests {
// Split the tar
let mut tar_cursor = Cursor::new(original_tar.clone());
let repo = create_test_repository().unwrap();
let mut writer = repo.create_stream(None, None);
let mut writer = repo.create_stream(TAR_LAYER_CONTENT_TYPE, None);
split(&mut tar_cursor, &mut writer).unwrap();
let object_id = writer.done().unwrap();

// Read back entries and compare with original headers
let mut reader: SplitStreamReader<std::fs::File, Sha256HashValue> =
SplitStreamReader::new(repo.open_object(&object_id).unwrap().into()).unwrap();
let mut reader: SplitStreamReader<std::fs::File, Sha256HashValue> = SplitStreamReader::new(
repo.open_object(&object_id).unwrap().into(),
Some(TAR_LAYER_CONTENT_TYPE),
)
.unwrap();
let mut entries = Vec::new();

while let Some(entry) = get_entry(&mut reader).unwrap() {
Expand Down
30 changes: 28 additions & 2 deletions crates/composefs/src/fsverity/hashvalue.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ use core::{fmt, hash::Hash};

use hex::FromHexError;
use sha2::{digest::FixedOutputReset, digest::Output, Digest, Sha256, Sha512};
use std::cmp::Ord;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout, Unaligned};

pub trait FsVerityHashValue
Expand All @@ -12,6 +13,7 @@ where
Self: Hash + Eq,
Self: fmt::Debug,
Self: Send + Sync + Unpin + 'static,
Self: PartialOrd + Ord,
{
type Digest: Digest + FixedOutputReset + fmt::Debug;
const ALGORITHM: u8;
Expand Down Expand Up @@ -93,7 +95,19 @@ impl fmt::Debug for Sha512HashValue {
}
}

#[derive(Clone, Eq, FromBytes, Hash, Immutable, IntoBytes, KnownLayout, PartialEq, Unaligned)]
#[derive(
Clone,
Eq,
FromBytes,
Hash,
Immutable,
IntoBytes,
KnownLayout,
PartialEq,
Unaligned,
PartialOrd,
Ord,
)]
#[repr(C)]
pub struct Sha256HashValue([u8; 32]);

Expand All @@ -110,7 +124,19 @@ impl FsVerityHashValue for Sha256HashValue {
const ID: &str = "sha256";
}

#[derive(Clone, Eq, FromBytes, Hash, Immutable, IntoBytes, KnownLayout, PartialEq, Unaligned)]
#[derive(
Clone,
Eq,
FromBytes,
Hash,
Immutable,
IntoBytes,
KnownLayout,
PartialEq,
Unaligned,
PartialOrd,
Ord,
)]
#[repr(C)]
pub struct Sha512HashValue([u8; 64]);

Expand Down
Loading
Loading