Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,14 @@
- move weak reference and downcasting examples into module docs
- expand module introduction describing use cases
- document rationale for separating `ByteSource` and `ByteOwner`
- added optional `winnow` feature for parser integration
- added `INVENTORY.md` for tracking future work and noted it in `AGENTS.md`
- documented safety rationale for `winnow` integration
- implemented `Stream` directly for `Bytes` with a safe `iter_offsets` iterator
- added `pop_back` and `pop_front` helpers and rewrote parser examples
- removed the Completed Work section from `INVENTORY.md` and documented its use
- rewrote `winnow::view` to use safe helpers and added `view_elems(count)` parser
- `winnow::view_elems` now returns a Parser closure for idiomatic usage
in a dedicated AGENTS section
- add tests for weak reference upgrade/downgrade and Kani proofs for view helpers
- add examples for quick start and PyBytes usage
Expand Down
2 changes: 2 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ ownedbytes = { version = "0.9.0", optional = true }
memmap2 = { version = "0.9.5", optional = true }
zerocopy = { version = "0.8.26", optional = true, features = ["derive"] }
pyo3 = { version = "0.25.1", optional = true }
winnow = { version = "0.7.12", optional = true }

[dev-dependencies]
proptest = "1.7"
Expand All @@ -23,6 +24,7 @@ ownedbytes = ["dep:ownedbytes"]
mmap = ["dep:memmap2"]
zerocopy = ["dep:zerocopy"]
pyo3 = ["dep:pyo3"]
winnow = ["dep:winnow"]

[lints.rust]
unexpected_cfgs = { level = "warn", check-cfg = ['cfg(kani)'] }
Expand Down
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ Other optional features provide additional integrations:
- `mmap` – enables memory-mapped file handling via the `memmap2` crate.
- `zerocopy` – exposes the [`view`](src/view.rs) module for typed zero-copy access and allows using `zerocopy` types as sources.
- `pyo3` – builds the [`pybytes`](src/pybytes.rs) module to provide Python bindings for `Bytes`.
- `winnow` – implements the [`Stream`](https://docs.rs/winnow/) traits for `Bytes` and offers parsers (`view`, `view_elems(count)`) that return typed `View`s.

Enabling the `pyo3` feature requires the Python development headers and libraries
(for example `libpython3.x`). Running `cargo test --all-features` therefore
Expand Down
14 changes: 14 additions & 0 deletions src/bytes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,20 @@ impl Bytes {
})
}

/// Removes and returns the first byte of `self`.
pub fn pop_front(&mut self) -> Option<u8> {
let (&b, rest) = self.data.split_first()?;
self.data = rest;
Some(b)
}

/// Removes and returns the last byte of `self`.
pub fn pop_back(&mut self) -> Option<u8> {
let (last, rest) = self.data.split_last()?;
self.data = rest;
Some(*last)
}

/// Create a weak pointer.
pub fn downgrade(&self) -> WeakBytes {
WeakBytes {
Expand Down
4 changes: 4 additions & 0 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@ pub mod view;
/// Python bindings for [`Bytes`].
pub mod pybytes;

#[cfg(feature = "winnow")]
/// Integration with the `winnow` parser library.
pub mod winnow;

#[cfg(test)]
mod tests;

Expand Down
50 changes: 50 additions & 0 deletions src/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,56 @@ fn test_weakview_clone_upgrade() {
assert!(weak_clone.upgrade().is_none());
}

#[cfg(feature = "winnow")]
#[test]
fn test_winnow_stream_take() {
use winnow::error::ContextError;
use winnow::stream::AsBytes;
use winnow::token::take;
use winnow::Parser;

let mut input = Bytes::from(vec![1u8, 2, 3, 4]);
let mut parser = take::<_, _, ContextError>(2usize);
let prefix: Bytes = parser.parse_next(&mut input).expect("take");
assert_eq!(prefix.as_ref(), [1u8, 2].as_ref());
assert_eq!(input.as_bytes(), [3u8, 4].as_ref());
}

#[cfg(all(feature = "winnow", feature = "zerocopy"))]
#[test]
fn test_winnow_view_parser() {
use winnow::error::ContextError;
use winnow::stream::AsBytes;
use winnow::Parser;
#[derive(zerocopy::TryFromBytes, zerocopy::KnownLayout, zerocopy::Immutable)]
#[repr(C)]
struct Pair {
a: u16,
b: u16,
}

let mut input = Bytes::from(vec![1u8, 0, 2, 0]);
let mut parser = crate::winnow::view::<Pair, ContextError>;
let view = parser.parse_next(&mut input).expect("view");
assert_eq!(view.a, 1);
assert_eq!(view.b, 2);
assert!(input.as_bytes().is_empty());
}

#[cfg(all(feature = "winnow", feature = "zerocopy"))]
#[test]
fn test_winnow_view_elems_parser() {
use winnow::error::ContextError;
use winnow::stream::AsBytes;
use winnow::Parser;

let mut input = Bytes::from(vec![1u8, 2, 3, 4]);
let mut parser = crate::winnow::view_elems::<[u8], ContextError>(3);
let view = parser.parse_next(&mut input).expect("view_elems");
assert_eq!(view.as_ref(), [1u8, 2, 3].as_ref());
assert_eq!(input.as_bytes(), [4u8].as_ref());
}

#[cfg(feature = "mmap")]
#[test]
fn test_mmap_mut_source() {
Expand Down
217 changes: 217 additions & 0 deletions src/winnow.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,217 @@
// Winnow integration for anybytes

use crate::Bytes;
use std::num::NonZeroUsize;
use winnow::error::{ErrMode, Needed, ParserError};
use winnow::stream::{
AsBytes, Compare, CompareResult, FindSlice, Offset, SliceLen, Stream, StreamIsPartial,
UpdateSlice,
};

#[cfg(feature = "zerocopy")]
use crate::view::View;
#[cfg(feature = "zerocopy")]
use zerocopy::{Immutable, KnownLayout, TryFromBytes};

/// Checkpoint for [`Bytes`] parsing with winnow.
#[derive(Clone, Debug)]
pub struct BytesCheckpoint(Bytes);

/// Iterator yielding `(offset, byte)` pairs for [`Bytes`].
#[derive(Clone, Debug)]
pub struct BytesIterOffsets {
bytes: Bytes,
offset: usize,
}

impl Iterator for BytesIterOffsets {
type Item = (usize, u8);

fn next(&mut self) -> Option<Self::Item> {
let token = self.bytes.pop_front()?;
let offset = self.offset;
self.offset += 1;
Some((offset, token))
}
}

impl SliceLen for Bytes {
#[inline(always)]
fn slice_len(&self) -> usize {
self.as_slice().len()
}
}

impl Stream for Bytes {
type Token = u8;
type Slice = Bytes;

type IterOffsets = BytesIterOffsets;

type Checkpoint = BytesCheckpoint;

#[inline(always)]
fn iter_offsets(&self) -> Self::IterOffsets {
BytesIterOffsets {
bytes: self.clone(),
offset: 0,
}
}

#[inline(always)]
fn eof_offset(&self) -> usize {
self.as_slice().len()
}

#[inline(always)]
fn next_token(&mut self) -> Option<Self::Token> {
self.pop_front()
}

#[inline(always)]
fn peek_token(&self) -> Option<Self::Token> {
self.as_slice().first().copied()
}

#[inline(always)]
fn offset_for<P>(&self, predicate: P) -> Option<usize>
where
P: Fn(Self::Token) -> bool,
{
self.as_slice().iter().position(|b| predicate(*b))
}

#[inline(always)]
fn offset_at(&self, tokens: usize) -> Result<usize, Needed> {
let remaining = self.as_slice().len();
if let Some(needed) = tokens.checked_sub(remaining).and_then(NonZeroUsize::new) {
Err(Needed::Size(needed))
} else {
Ok(tokens)
}
}

#[inline(always)]
fn next_slice(&mut self, offset: usize) -> Self::Slice {
self.take_prefix(offset).expect("offset within bounds")
}

#[inline(always)]
fn peek_slice(&self, offset: usize) -> Self::Slice {
self.slice(..offset)
}

#[inline(always)]
fn checkpoint(&self) -> Self::Checkpoint {
BytesCheckpoint(self.clone())
}

#[inline(always)]
fn reset(&mut self, checkpoint: &Self::Checkpoint) {
*self = checkpoint.0.clone();
}

#[allow(deprecated)]
#[inline(always)]
fn raw(&self) -> &dyn core::fmt::Debug {
self
}
}

impl StreamIsPartial for Bytes {
type PartialState = ();

#[inline]
fn complete(&mut self) -> Self::PartialState {}

#[inline]
fn restore_partial(&mut self, _state: Self::PartialState) {}

#[inline(always)]
fn is_partial_supported() -> bool {
false
}
}

impl Offset for Bytes {
#[inline(always)]
fn offset_from(&self, start: &Self) -> usize {
let self_ptr = self.as_slice().as_ptr() as usize;
let start_ptr = start.as_slice().as_ptr() as usize;
self_ptr - start_ptr
}
}

impl Offset<BytesCheckpoint> for Bytes {
#[inline(always)]
fn offset_from(&self, other: &BytesCheckpoint) -> usize {
self.offset_from(&other.0)
}
}

impl Offset for BytesCheckpoint {
#[inline(always)]
fn offset_from(&self, start: &Self) -> usize {
self.0.offset_from(&start.0)
}
}

impl AsBytes for Bytes {
#[inline(always)]
fn as_bytes(&self) -> &[u8] {
self.as_slice()
}
}

impl<T> Compare<T> for Bytes
where
for<'a> &'a [u8]: Compare<T>,
{
#[inline(always)]
fn compare(&self, t: T) -> CompareResult {
self.as_slice().compare(t)
}
}

impl<S> FindSlice<S> for Bytes
where
for<'a> &'a [u8]: FindSlice<S>,
{
#[inline(always)]
fn find_slice(&self, substr: S) -> Option<core::ops::Range<usize>> {
self.as_slice().find_slice(substr)
}
}

impl UpdateSlice for Bytes {
#[inline(always)]
fn update_slice(self, inner: Self::Slice) -> Self {
inner
}
}

#[cfg(feature = "zerocopy")]
/// Parse a `View` of `T` from the beginning of the input.
pub fn view<T, E>(input: &mut Bytes) -> Result<View<T>, ErrMode<E>>
where
T: ?Sized + TryFromBytes + KnownLayout + Immutable,
E: ParserError<Bytes>,
{
input
.view_prefix::<T>()
.map_err(|_| ErrMode::Backtrack(E::from_input(input)))
}

#[cfg(feature = "zerocopy")]
/// Return a parser producing a slice-like `View` with `count` elements.
pub fn view_elems<T, E>(count: usize) -> impl winnow::Parser<Bytes, View<T>, ErrMode<E>>
where
T: ?Sized + TryFromBytes + KnownLayout<PointerMetadata = usize> + Immutable,
E: ParserError<Bytes>,
{
move |input: &mut Bytes| {
input
.view_prefix_with_elems::<T>(count)
.map_err(|_| ErrMode::Backtrack(E::from_input(input)))
}
}