Skip to content

Commit af8f463

Browse files
alexlarssonalexlarsson
committed
splitstream: Add content_type to header
This is an extra check to avoid confusion.
1 parent ef70c5d commit af8f463

File tree

7 files changed

+43
-20
lines changed

7 files changed

+43
-20
lines changed

‎crates/cfsctl/src/main.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,7 @@ async fn main() -> Result<()> {
180180
}
181181
}
182182
Command::Cat { name } => {
183-
repo.merge_splitstream(&name, None, &mut std::io::stdout())?;
183+
repo.merge_splitstream(&name, None, None, &mut std::io::stdout())?;
184184
}
185185
Command::ImportImage { reference } => {
186186
let image_id = repo.import_image(&reference, &mut std::io::stdin())?;

‎crates/composefs-http/src/lib.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ impl<ObjectID: FsVerityHashValue> Downloader<ObjectID> {
6161
}
6262

6363
fn open_splitstream(&self, id: &ObjectID) -> Result<SplitStreamReader<File, ObjectID>> {
64-
SplitStreamReader::new(File::from(self.repo.open_object(id)?))
64+
SplitStreamReader::new(File::from(self.repo.open_object(id)?), None)
6565
}
6666

6767
fn read_object(&self, id: &ObjectID) -> Result<Vec<u8>> {

‎crates/composefs-oci/src/image.rs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ use composefs::{
99
tree::{Directory, FileSystem, Inode, Leaf},
1010
};
1111

12+
use crate::skopeo::{OCI_CONFIG_CONTENT_TYPE, TAR_LAYER_CONTENT_TYPE};
1213
use crate::tar::{TarEntry, TarItem};
1314

1415
pub fn process_entry<ObjectID: FsVerityHashValue>(
@@ -74,14 +75,14 @@ pub fn create_filesystem<ObjectID: FsVerityHashValue>(
7475
) -> Result<FileSystem<ObjectID>> {
7576
let mut filesystem = FileSystem::default();
7677

77-
let mut config_stream = repo.open_stream(config_name, config_verity)?;
78+
let mut config_stream = repo.open_stream(config_name, config_verity, Some(OCI_CONFIG_CONTENT_TYPE))?;
7879
let config = ImageConfiguration::from_reader(&mut config_stream)?;
7980

8081
for diff_id in config.rootfs().diff_ids() {
8182
let layer_sha256 = super::sha256_from_digest(diff_id)?;
8283
let layer_verity = config_stream.lookup(&layer_sha256)?;
8384

84-
let mut layer_stream = repo.open_stream(&hex::encode(layer_sha256), Some(layer_verity))?;
85+
let mut layer_stream = repo.open_stream(&hex::encode(layer_sha256), Some(layer_verity), Some(TAR_LAYER_CONTENT_TYPE))?;
8586
while let Some(entry) = crate::tar::get_entry(&mut layer_stream)? {
8687
process_entry(&mut filesystem, entry)?;
8788
}

‎crates/composefs-oci/src/lib.rs

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ use composefs::{
1616
};
1717

1818
use crate::tar::get_entry;
19+
use crate::skopeo::{OCI_CONFIG_CONTENT_TYPE, TAR_LAYER_CONTENT_TYPE};
1920

2021
type ContentAndVerity<ObjectID> = (Sha256Digest, ObjectID);
2122

@@ -39,14 +40,14 @@ pub fn import_layer<ObjectID: FsVerityHashValue>(
3940
name: Option<&str>,
4041
tar_stream: &mut impl Read,
4142
) -> Result<ObjectID> {
42-
repo.ensure_stream(sha256, |writer| tar::split(tar_stream, writer), name)
43+
repo.ensure_stream(sha256, TAR_LAYER_CONTENT_TYPE, |writer| tar::split(tar_stream, writer), name)
4344
}
4445

4546
pub fn ls_layer<ObjectID: FsVerityHashValue>(
4647
repo: &Repository<ObjectID>,
4748
name: &str,
4849
) -> Result<()> {
49-
let mut split_stream = repo.open_stream(name, None)?;
50+
let mut split_stream = repo.open_stream(name, None, Some(TAR_LAYER_CONTENT_TYPE))?;
5051

5152
while let Some(entry) = get_entry(&mut split_stream)? {
5253
println!("{entry}");
@@ -81,7 +82,7 @@ pub fn open_config<ObjectID: FsVerityHashValue>(
8182
.with_context(|| format!("Object {name} is unknown to us"))?
8283
}
8384
};
84-
let mut stream = repo.open_stream(name, Some(id))?;
85+
let mut stream = repo.open_stream(name, Some(id), Some(OCI_CONFIG_CONTENT_TYPE))?;
8586
let config = ImageConfiguration::from_reader(&mut stream)?;
8687
Ok((config, stream.get_mappings()))
8788
}
@@ -104,7 +105,7 @@ pub fn open_config_shallow<ObjectID: FsVerityHashValue>(
104105
// we need to manually check the content digest
105106
let expected_hash = parse_sha256(name)
106107
.context("Containers must be referred to by sha256 if verity is missing")?;
107-
let mut stream = repo.open_stream(name, None)?;
108+
let mut stream = repo.open_stream(name, None, Some(OCI_CONFIG_CONTENT_TYPE))?;
108109
let mut raw_config = vec![];
109110
stream.read_to_end(&mut raw_config)?;
110111
ensure!(hash(&raw_config) == expected_hash, "Data integrity issue");
@@ -121,7 +122,7 @@ pub fn write_config<ObjectID: FsVerityHashValue>(
121122
let json = config.to_string()?;
122123
let json_bytes = json.as_bytes();
123124
let sha256 = hash(json_bytes);
124-
let mut stream = repo.create_stream(Some(sha256));
125+
let mut stream = repo.create_stream(OCI_CONFIG_CONTENT_TYPE, Some(sha256));
125126
stream.add_sha256_mappings(refs);
126127
stream.write_inline(json_bytes);
127128
let id = repo.write_stream(stream, None)?;

‎crates/composefs-oci/src/skopeo.rs

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@ use composefs::{
1616

1717
use crate::{sha256_from_descriptor, sha256_from_digest, tar::split_async, ContentAndVerity};
1818

19+
pub const TAR_LAYER_CONTENT_TYPE : u64 = 0x2a037edfcae1ffea;
20+
pub const OCI_CONFIG_CONTENT_TYPE : u64 = 0x44218c839727a80b;
21+
1922
struct ImageOp<ObjectID: FsVerityHashValue> {
2023
repo: Arc<Repository<ObjectID>>,
2124
proxy: ImageProxy,
@@ -78,7 +81,7 @@ impl<ObjectID: FsVerityHashValue> ImageOp<ObjectID> {
7881
self.progress
7982
.println(format!("Fetching layer {}", hex::encode(layer_sha256)))?;
8083

81-
let mut splitstream = self.repo.create_stream(Some(layer_sha256));
84+
let mut splitstream = self.repo.create_stream(TAR_LAYER_CONTENT_TYPE, Some(layer_sha256));
8285
match descriptor.media_type() {
8386
MediaType::ImageLayer => {
8487
split_async(progress, &mut splitstream).await?;
@@ -157,7 +160,7 @@ impl<ObjectID: FsVerityHashValue> ImageOp<ObjectID> {
157160

158161
let mut splitstream = self
159162
.repo
160-
.create_stream(Some(config_sha256));
163+
.create_stream(OCI_CONFIG_CONTENT_TYPE, Some(config_sha256));
161164

162165
// Collect the results.
163166
for (layer_sha256, future) in entries {

‎crates/composefs/src/repository.rs

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -184,9 +184,10 @@ impl<ObjectID: FsVerityHashValue> Repository<ObjectID> {
184184
/// store the result.
185185
pub fn create_stream(
186186
self: &Arc<Self>,
187+
content_type: u64,
187188
sha256: Option<Sha256Digest>,
188189
) -> SplitStreamWriter<ObjectID> {
189-
SplitStreamWriter::new(self, sha256)
190+
SplitStreamWriter::new(self, content_type, sha256)
190191
}
191192

192193
fn format_object_path(id: &ObjectID) -> String {
@@ -223,7 +224,7 @@ impl<ObjectID: FsVerityHashValue> Repository<ObjectID> {
223224
Ok(stream) => {
224225
let measured_verity: ObjectID = measure_verity(&stream)?;
225226
let mut context = Sha256::new();
226-
let mut split_stream = SplitStreamReader::new(File::from(stream))?;
227+
let mut split_stream = SplitStreamReader::new(File::from(stream), None)?;
227228

228229
// check the verity of all linked streams
229230
for (body, verity) in split_stream.iter_mappings() {
@@ -296,6 +297,7 @@ impl<ObjectID: FsVerityHashValue> Repository<ObjectID> {
296297
pub fn ensure_stream(
297298
self: &Arc<Self>,
298299
sha256: &Sha256Digest,
300+
content_type: u64,
299301
callback: impl FnOnce(&mut SplitStreamWriter<ObjectID>) -> Result<()>,
300302
reference: Option<&str>,
301303
) -> Result<ObjectID> {
@@ -304,7 +306,7 @@ impl<ObjectID: FsVerityHashValue> Repository<ObjectID> {
304306
let object_id = match self.has_stream(sha256)? {
305307
Some(id) => id,
306308
None => {
307-
let mut writer = self.create_stream(Some(*sha256));
309+
let mut writer = self.create_stream(content_type, Some(*sha256));
308310
callback(&mut writer)?;
309311
let object_id = writer.done()?;
310312

@@ -326,6 +328,7 @@ impl<ObjectID: FsVerityHashValue> Repository<ObjectID> {
326328
&self,
327329
name: &str,
328330
verity: Option<&ObjectID>,
331+
expected_content_type: Option<u64>,
329332
) -> Result<SplitStreamReader<File, ObjectID>> {
330333
let filename = format!("streams/{name}");
331334

@@ -337,7 +340,7 @@ impl<ObjectID: FsVerityHashValue> Repository<ObjectID> {
337340
.with_context(|| format!("Opening ref 'streams/{name}'"))?
338341
});
339342

340-
SplitStreamReader::new(file)
343+
SplitStreamReader::new(file, expected_content_type)
341344
}
342345

343346
pub fn open_object(&self, id: &ObjectID) -> Result<OwnedFd> {
@@ -348,9 +351,10 @@ impl<ObjectID: FsVerityHashValue> Repository<ObjectID> {
348351
&self,
349352
name: &str,
350353
verity: Option<&ObjectID>,
354+
expected_content_type: Option<u64>,
351355
stream: &mut impl Write,
352356
) -> Result<()> {
353-
let mut split_stream = self.open_stream(name, verity)?;
357+
let mut split_stream = self.open_stream(name, verity, expected_content_type)?;
354358
split_stream.cat(stream, |id| -> Result<Vec<u8>> {
355359
let mut data = vec![];
356360
File::from(self.open_object(id)?).read_to_end(&mut data)?;
@@ -553,7 +557,7 @@ impl<ObjectID: FsVerityHashValue> Repository<ObjectID> {
553557
println!("{object:?} lives as a stream");
554558
objects.insert(object.clone());
555559

556-
let mut split_stream = self.open_stream(&object.to_hex(), None)?;
560+
let mut split_stream = self.open_stream(&object.to_hex(), None, None)?;
557561
split_stream.get_object_refs(|id| {
558562
println!(" with {id:?}");
559563
objects.insert(id.clone());

‎crates/composefs/src/splitstream.rs

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ pub const SPLITSTREAM_MAGIC : [u8; 7] = [b'S', b'p', b'l', b't', b'S', b't', b'r
2626
pub struct SplitstreamHeader {
2727
pub magic: [u8; 7], // Contains SPLITSTREAM_MAGIC
2828
pub algorithm: u8,
29+
pub content_type: u64, // User can put whatever magic identifier they want there
2930
pub total_size: u64, // total size of inline chunks and external chunks
3031
pub n_refs: u64,
3132
pub n_mappings: u64,
@@ -92,6 +93,7 @@ pub struct SplitStreamWriter<ObjectID: FsVerityHashValue> {
9293
inline_content: Vec<u8>,
9394
total_size: u64,
9495
writer: Encoder<'static, Vec<u8>>,
96+
pub content_type: u64,
9597
pub sha256: Option<(Sha256, Sha256Digest)>,
9698
}
9799

@@ -109,13 +111,15 @@ impl<ObjectID: FsVerityHashValue> std::fmt::Debug for SplitStreamWriter<ObjectID
109111
impl<ObjectID: FsVerityHashValue> SplitStreamWriter<ObjectID> {
110112
pub fn new(
111113
repo: &Arc<Repository<ObjectID>>,
114+
content_type: u64,
112115
sha256: Option<Sha256Digest>,
113116
) -> Self {
114117
// SAFETY: we surely can't get an error writing the header to a Vec<u8>
115118
let writer = Encoder::new(vec![], 0).unwrap();
116119

117120
Self {
118121
repo: Arc::clone(repo),
122+
content_type: content_type,
119123
inline_content: vec![],
120124
refs: vec![],
121125
total_size: 0,
@@ -218,6 +222,7 @@ impl<ObjectID: FsVerityHashValue> SplitStreamWriter<ObjectID> {
218222
let header = SplitstreamHeader {
219223
magic: SPLITSTREAM_MAGIC,
220224
algorithm: ObjectID::ALGORITHM,
225+
content_type: self.content_type,
221226
total_size: u64::to_le(self.total_size),
222227
n_refs: u64::to_le(self.refs.len() as u64),
223228
n_mappings: u64::to_le(self.mappings.map.len() as u64),
@@ -252,6 +257,7 @@ pub enum SplitStreamData<ObjectID: FsVerityHashValue> {
252257
pub struct SplitStreamReader<R: Read, ObjectID: FsVerityHashValue> {
253258
decoder: Decoder<'static, BufReader<R>>,
254259
inline_bytes: usize,
260+
pub content_type: u64,
255261
pub total_size: u64,
256262
pub refs: Vec<ObjectID>,
257263
mappings: Vec<MappingEntry>,
@@ -293,17 +299,24 @@ enum ChunkType<ObjectID: FsVerityHashValue> {
293299
}
294300

295301
impl<R: Read, ObjectID: FsVerityHashValue> SplitStreamReader<R, ObjectID> {
296-
pub fn new(mut reader: R) -> Result<Self> {
302+
pub fn new(mut reader: R, expected_content_type: Option<u64>) -> Result<Self> {
297303

298304
let header = SplitstreamHeader::read_from_io(&mut reader)
299305
.map_err(|e| Error::msg(format!("Error reading splitstream header: {:?}", e)))?;
300306

301307
if header.magic != SPLITSTREAM_MAGIC {
302-
bail!("Invalida splitstream header magic value");
308+
bail!("Invalid splitstream header magic value");
303309
}
304310

305311
if header.algorithm != ObjectID::ALGORITHM {
306-
bail!("Invalida splitstream algorithm type");
312+
bail!("Invalid splitstream algorithm type");
313+
}
314+
315+
let content_type = u64::from_le(header.content_type);
316+
if let Some(expected) = expected_content_type {
317+
if content_type != expected {
318+
bail!("Invalid splitstream content type");
319+
}
307320
}
308321

309322
let total_size = u64::from_le(header.total_size);
@@ -333,6 +346,7 @@ impl<R: Read, ObjectID: FsVerityHashValue> SplitStreamReader<R, ObjectID> {
333346
Ok(Self {
334347
decoder,
335348
inline_bytes: 0,
349+
content_type,
336350
total_size,
337351
refs,
338352
mappings,

0 commit comments

Comments
 (0)