Merge pull request #58 from triblespace/codex/add-section-tracking-to-bytearea

Add section handles for ByteArea

Author: somethingelseentirely

CHANGELOG.md

@@ -4,6 +4,7 @@
 - added example demonstrating `ByteArea` with multiple typed sections, concurrent mutations, and freezing or persisting the area
 - added example combining Python bindings with winnow parsing
 - added Python example demonstrating structured parsing with winnow's `view`
+- added `SectionHandle` for reconstructing sections from a frozen `ByteArea`
 - added `ByteSource` support for `VecDeque<T>` when `zerocopy` is enabled and kept the deque as owner
 - added `ByteSource` support for `Cow<'static, T>` where `T: AsRef<[u8]>`
 - added `ByteArea` for staged file writes with `Section::freeze()` to return `Bytes`

README.md

@@ -103,7 +103,8 @@ To map only a portion of a file use the unsafe helper
 
 ### Byte Area
 
-Use `ByteArea` to incrementally build immutable bytes on disk:
+Use `ByteArea` to incrementally build immutable bytes on disk; each section can
+yield a handle that reconstructs its range after the area is frozen:
 
 ```rust
 use anybytes::area::ByteArea;
@@ -112,11 +113,12 @@ let mut area = ByteArea::new().unwrap();
 let mut sections = area.sections();
 let mut section = sections.reserve::<u8>(4).unwrap();
 section.copy_from_slice(b"test");
+let handle = section.handle();
 let bytes = section.freeze().unwrap();
-assert_eq!(bytes.as_ref(), b"test".as_ref());
 drop(sections);
 let all = area.freeze().unwrap();
-assert_eq!(all.as_ref(), b"test".as_ref());
+assert_eq!(handle.bytes(&all).as_ref(), bytes.as_ref());
+assert_eq!(handle.view(&all).unwrap().as_ref(), b"test".as_ref());
 ```
 
 Call `area.persist(path)` to keep the temporary file instead of mapping it.

examples/byte_area.rs

@@ -12,6 +12,10 @@ fn main() -> std::io::Result<()> {
     raw.as_mut_slice().copy_from_slice(b"test");
     nums.as_mut_slice().copy_from_slice(&[1, 2]);
 
+    // Store handles so we can later extract the sections from the frozen area.
+    let handle_raw = raw.handle();
+    let handle_nums = nums.handle();
+
     // Freeze the sections into immutable `Bytes`.
     let frozen_raw: Bytes = raw.freeze()?;
     let frozen_nums: Bytes = nums.freeze()?;
@@ -25,8 +29,8 @@ fn main() -> std::io::Result<()> {
     if memory_or_file {
         // Freeze the whole area into immutable `Bytes`.
         let all: Bytes = area.freeze()?;
-        assert_eq!(&all[..4], b"test");
-        assert_eq!(all.slice(4..).view::<[u32]>().unwrap().as_ref(), &[1, 2]);
+        assert_eq!(handle_raw.bytes(&all).as_ref(), frozen_raw.as_ref());
+        assert_eq!(handle_nums.view(&all).unwrap().as_ref(), &[1, 2]);
     } else {
         // Persist the temporary file.
         let dir = tempdir()?;

src/area.rs

@@ -26,17 +26,19 @@
 //!
 //! let mut a = sections.reserve::<u8>(1).unwrap();
 //! a.as_mut_slice()[0] = 1;
+//! let handle_a = a.handle();
 //!
 //! let mut b = sections.reserve::<u32>(1).unwrap();
 //! b.as_mut_slice()[0] = 2;
+//! let handle_b = b.handle();
 //!
 //! let bytes_a = a.freeze().unwrap();
 //! let bytes_b = b.freeze().unwrap();
 //! drop(sections);
 //! let all = area.freeze().unwrap();
 //!
-//! assert_eq!(bytes_a.as_ref(), &[1]);
-//! assert_eq!(bytes_b.as_ref(), &2u32.to_ne_bytes());
+//! assert_eq!(handle_a.bytes(&all).as_ref(), &[1]);
+//! assert_eq!(handle_b.bytes(&all).as_ref(), &2u32.to_ne_bytes());
 //!
 //! let mut expected = Vec::new();
 //! expected.extend_from_slice(&[1]);
@@ -58,6 +60,9 @@ use crate::Bytes;
 #[cfg(feature = "zerocopy")]
 use zerocopy::{FromBytes, Immutable};
 
+#[cfg(feature = "zerocopy")]
+use crate::view::{View, ViewError};
+
 /// Alignment helper.
 fn align_up(val: usize, align: usize) -> usize {
     (val + align - 1) & !(align - 1)
@@ -135,6 +140,7 @@ impl<'area> SectionWriter<'area> {
         Ok(Section {
             mmap,
             offset,
+            start,
             elems,
             _marker: PhantomData,
         })
@@ -148,6 +154,8 @@ pub struct Section<'arena, T> {
     mmap: memmap2::MmapMut,
     /// Offset from the beginning of `mmap` to the start of the buffer.
     offset: usize,
+    /// Absolute start offset within the [`ByteArea`] in bytes.
+    start: usize,
     /// Number of elements in the buffer.
     elems: usize,
     /// Marker tying the section to the area and element type.
@@ -176,6 +184,15 @@ where
         let map = self.mmap.make_read_only()?;
         Ok(Bytes::from_source(map).slice(offset..offset + len_bytes))
     }
+
+    /// Return a handle that can reconstruct this section from a frozen [`ByteArea`].
+    pub fn handle(&self) -> SectionHandle<T> {
+        SectionHandle {
+            offset: self.start,
+            len: self.elems * core::mem::size_of::<T>(),
+            _type: PhantomData,
+        }
+    }
 }
 
 impl<'arena, T> core::ops::Deref for Section<'arena, T>
@@ -221,3 +238,29 @@ where
         self
     }
 }
+
+/// Handle referencing a [`Section`] within a frozen [`ByteArea`].
+#[derive(Clone, Copy, Debug)]
+pub struct SectionHandle<T> {
+    /// Absolute byte offset from the start of the area.
+    pub offset: usize,
+    /// Length of the section in bytes.
+    pub len: usize,
+    /// Marker for the element type stored in the section.
+    _type: PhantomData<T>,
+}
+
+impl<T> SectionHandle<T>
+where
+    T: FromBytes + Immutable,
+{
+    /// Extract the raw bytes for this section from `area`.
+    pub fn bytes<'a>(&self, area: &'a Bytes) -> Bytes {
+        area.slice(self.offset..self.offset + self.len)
+    }
+
+    /// Interpret the section as a typed [`View`].
+    pub fn view(&self, area: &Bytes) -> Result<View<[T]>, ViewError> {
+        self.bytes(area).view::<[T]>()
+    }
+}

tests/area.rs

@@ -73,3 +73,27 @@ proptest! {
         prop_assert_eq!(all.as_ref(), expected.as_slice());
     }
 }
+
+#[test]
+fn handles_reconstruct_sections() {
+    let mut area = ByteArea::new().expect("area");
+    let mut sections = area.sections();
+
+    let mut a = sections.reserve::<u8>(1).expect("reserve u8");
+    a.as_mut_slice()[0] = 1;
+    let handle_a = a.handle();
+    let bytes_a = a.freeze().expect("freeze a");
+
+    let mut b = sections.reserve::<u32>(1).expect("reserve u32");
+    b.as_mut_slice()[0] = 2;
+    let handle_b = b.handle();
+    let bytes_b = b.freeze().expect("freeze b");
+
+    drop(sections);
+    let all = area.freeze().expect("freeze area");
+
+    assert_eq!(handle_a.bytes(&all).as_ref(), bytes_a.as_ref());
+    assert_eq!(handle_b.bytes(&all).as_ref(), bytes_b.as_ref());
+    assert_eq!(handle_a.view(&all).unwrap().as_ref(), &[1]);
+    assert_eq!(handle_b.view(&all).unwrap().as_ref(), &[2]);
+}