Optimize Ref<_, T> methods when T: Sized (#2752)

Benchmarks are surprisingly showing minimal impact...

With optimization:
test r#ref::tests::bench_deref_mut_sized  ... bench:           0.23 ns/iter (+/- 0.03)
test r#ref::tests::bench_deref_sized      ... bench:           0.44 ns/iter (+/- 0.04)
test r#ref::tests::bench_from_bytes_sized ... bench:           4.64 ns/iter (+/- 0.03)
test r#ref::tests::bench_into_mut_sized   ... bench:           0.23 ns/iter (+/- 0.03)
test r#ref::tests::bench_into_ref_sized   ... bench:           0.44 ns/iter (+/- 0.00)

Without optimization:
test r#ref::tests::bench_deref_mut_sized  ... bench:           0.22 ns/iter (+/- 0.02)
test r#ref::tests::bench_deref_sized      ... bench:           0.44 ns/iter (+/- 0.03)
test r#ref::tests::bench_from_bytes_sized ... bench:           4.66 ns/iter (+/- 0.31)
test r#ref::tests::bench_into_mut_sized   ... bench:           0.23 ns/iter (+/- 0.03)
test r#ref::tests::bench_into_ref_sized   ... bench:           0.44 ns/iter (+/- 0.01)

Closes #2752

Co-authored-by: Joshua Liebow-Feeser <[email protected]>

Author: bmberger

src/lib.rs

@@ -334,13 +334,17 @@
     __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS,
     feature(layout_for_ptr, coverage_attribute)
 )]
+#![cfg_attr(test, feature(test))]
 
 // This is a hack to allow zerocopy-derive derives to work in this crate. They
 // assume that zerocopy is linked as an extern crate, so they access items from
 // it as `zerocopy::Xxx`. This makes that still work.
 #[cfg(any(feature = "derive", test))]
 extern crate self as zerocopy;
 
+#[cfg(test)]
+extern crate test;
+
 #[doc(hidden)]
 #[macro_use]
 pub mod util;

src/ref.rs

@@ -616,6 +616,36 @@ where
         // SAFETY: We don't call any methods on `b` other than those provided by
         // `IntoByteSlice`.
         let b = unsafe { r.into_byte_slice() };
+        let b = b.into_byte_slice();
+
+        if let crate::layout::SizeInfo::Sized { .. } = T::LAYOUT.size_info {
+            let ptr = Ptr::from_ref(b);
+            // SAFETY: `T::raw_from_ptr_len` promises to preserve address and provenance in the referent
+            let ptr: Ptr<'_, T, _> = unsafe {
+                ptr.transmute_unchecked(|ptr| {
+                    let ptr = T::raw_from_ptr_len(
+                        ptr.as_non_null().cast(),
+                        <T::PointerMetadata as crate::PointerMetadata>::from_elem_count(0),
+                    );
+                    // SAFETY: The safety invariants of `Ptr::new` (see definition) are
+                    // satisfied:
+                    // 0. If `ptr`'s referent is not zero sized, then `ptr` has valid
+                    //    provenance for its referent, because it derived from `self`
+                    //    using a series of provenance-preserving operations, and
+                    //    because `self` has valid provenance for its referent. By the
+                    //    same argument, `ptr`'s referent is entirely contained within
+                    //    the same allocated object as `self`'s referent.
+                    // 1. If `ptr`'s referent is not zero sized, then the allocation of
+                    //    `ptr` is guaranteed to live for at least `'a`, because `ptr`
+                    //    is entirely contained in `self`, which lives for at least `'a`
+                    //    by invariant on `Ptr`.
+                    unsafe { crate::pointer::PtrInner::new(ptr) }
+                })
+            };
+            // SAFETY: By invariant on `r`, we know that size and alignment are valid.
+            let ptr = unsafe { ptr.assume_alignment() };
+            return ptr.as_ref();
+        }
 
         // PANICS: By post-condition on `into_byte_slice`, `b`'s size and
         // alignment are valid for `T`. By post-condition, `b.into_byte_slice()`
@@ -650,6 +680,36 @@ where
         // SAFETY: We don't call any methods on `b` other than those provided by
         // `IntoByteSliceMut`.
         let b = unsafe { r.into_byte_slice_mut() };
+        let b = b.into_byte_slice_mut();
+
+        if let crate::layout::SizeInfo::Sized { .. } = T::LAYOUT.size_info {
+            let ptr = Ptr::from_mut(b);
+            // SAFETY: `T::raw_from_ptr_len` promises to preserve address and provenance in the referent
+            let ptr: Ptr<'_, T, _> = unsafe {
+                ptr.transmute_unchecked(|ptr| {
+                    let ptr = T::raw_from_ptr_len(
+                        ptr.as_non_null().cast(),
+                        <T::PointerMetadata as crate::PointerMetadata>::from_elem_count(0),
+                    );
+                    // SAFETY: The safety invariants of `Ptr::new` (see definition) are
+                    // satisfied:
+                    // 0. If `ptr`'s referent is not zero sized, then `ptr` has valid
+                    //    provenance for its referent, because it derived from `self`
+                    //    using a series of provenance-preserving operations, and
+                    //    because `self` has valid provenance for its referent. By the
+                    //    same argument, `ptr`'s referent is entirely contained within
+                    //    the same allocated object as `self`'s referent.
+                    // 1. If `ptr`'s referent is not zero sized, then the allocation of
+                    //    `ptr` is guaranteed to live for at least `'a`, because `ptr`
+                    //    is entirely contained in `self`, which lives for at least `'a`
+                    //    by invariant on `Ptr`.
+                    unsafe { crate::pointer::PtrInner::new(ptr) }
+                })
+            };
+            // SAFETY: By invariant on `r`, we know that size and alignment are valid.
+            let ptr = unsafe { ptr.assume_alignment() };
+            return ptr.as_mut();
+        }
 
         // PANICS: By post-condition on `into_byte_slice_mut`, `b`'s size and
         // alignment are valid for `T`. By post-condition,
@@ -763,11 +823,41 @@ where
         // SAFETY: We don't call any methods on `b` other than those provided by
         // `ByteSlice`.
         let b = unsafe { self.as_byte_slice() };
+        let b = b.deref();
+
+        if let crate::layout::SizeInfo::Sized { .. } = T::LAYOUT.size_info {
+            let ptr = Ptr::from_ref(b);
+            // SAFETY: `T::raw_from_ptr_len` promises to preserve address and provenance in the referent
+            let ptr: Ptr<'_, T, _> = unsafe {
+                ptr.transmute_unchecked(|ptr| {
+                    let ptr = T::raw_from_ptr_len(
+                        ptr.as_non_null().cast(),
+                        <T::PointerMetadata as crate::PointerMetadata>::from_elem_count(0),
+                    );
+                    // SAFETY: The safety invariants of `Ptr::new` (see definition) are
+                    // satisfied:
+                    // 0. If `ptr`'s referent is not zero sized, then `ptr` has valid
+                    //    provenance for its referent, because it derived from `self`
+                    //    using a series of provenance-preserving operations, and
+                    //    because `self` has valid provenance for its referent. By the
+                    //    same argument, `ptr`'s referent is entirely contained within
+                    //    the same allocated object as `self`'s referent.
+                    // 1. If `ptr`'s referent is not zero sized, then the allocation of
+                    //    `ptr` is guaranteed to live for at least `'a`, because `ptr`
+                    //    is entirely contained in `self`, which lives for at least `'a`
+                    //    by invariant on `Ptr`.
+                    unsafe { crate::pointer::PtrInner::new(ptr) }
+                })
+            };
+            // SAFETY: By invariant on `self`, we know that size and alignment are valid.
+            let ptr = unsafe { ptr.assume_alignment() };
+            return ptr.as_ref();
+        }
 
         // PANICS: By postcondition on `as_byte_slice`, `b`'s size and alignment
         // are valid for `T`, and by invariant on `ByteSlice`, these are
         // preserved through `.deref()`, so this `unwrap` will not panic.
-        let ptr = Ptr::from_ref(b.deref())
+        let ptr = Ptr::from_ref(b)
             .try_cast_into_no_leftover::<T, BecauseImmutable>(None)
             .expect("zerocopy internal error: Deref::deref should be infallible");
         let ptr = ptr.recall_validity();
@@ -791,12 +881,42 @@ where
         // SAFETY: We don't call any methods on `b` other than those provided by
         // `ByteSliceMut`.
         let b = unsafe { self.as_byte_slice_mut() };
+        let b = b.deref_mut();
+
+        if let crate::layout::SizeInfo::Sized { .. } = T::LAYOUT.size_info {
+            let ptr = Ptr::from_mut(b);
+            // SAFETY: `T::raw_from_ptr_len` promises to preserve address and provenance in the referent
+            let ptr: Ptr<'_, T, _> = unsafe {
+                ptr.transmute_unchecked(|ptr| {
+                    let ptr = T::raw_from_ptr_len(
+                        ptr.as_non_null().cast(),
+                        <T::PointerMetadata as crate::PointerMetadata>::from_elem_count(0),
+                    );
+                    // SAFETY: The safety invariants of `Ptr::new` (see definition) are
+                    // satisfied:
+                    // 0. If `ptr`'s referent is not zero sized, then `ptr` has valid
+                    //    provenance for its referent, because it derived from `self`
+                    //    using a series of provenance-preserving operations, and
+                    //    because `self` has valid provenance for its referent. By the
+                    //    same argument, `ptr`'s referent is entirely contained within
+                    //    the same allocated object as `self`'s referent.
+                    // 1. If `ptr`'s referent is not zero sized, then the allocation of
+                    //    `ptr` is guaranteed to live for at least `'a`, because `ptr`
+                    //    is entirely contained in `self`, which lives for at least `'a`
+                    //    by invariant on `Ptr`.
+                    unsafe { crate::pointer::PtrInner::new(ptr) }
+                })
+            };
+            // SAFETY: By invariant on `self`, we know that size and alignment are valid.
+            let ptr = unsafe { ptr.assume_alignment() };
+            return ptr.as_mut();
+        }
 
         // PANICS: By postcondition on `as_byte_slice_mut`, `b`'s size and
         // alignment are valid for `T`, and by invariant on `ByteSlice`, these
         // are preserved through `.deref_mut()`, so this `unwrap` will not
         // panic.
-        let ptr = Ptr::from_mut(b.deref_mut())
+        let ptr = Ptr::from_mut(b)
             .try_cast_into_no_leftover::<T, BecauseExclusive>(None)
             .expect("zerocopy internal error: DerefMut::deref_mut should be infallible");
         let ptr = ptr.recall_validity::<_, (_, (_, (BecauseExclusive, BecauseExclusive)))>();
@@ -879,6 +999,48 @@ mod tests {
 
     use super::*;
     use crate::util::testutil::*;
+    use test::{self, Bencher};
+
+    #[bench]
+    fn bench_from_bytes_sized(b: &mut Bencher) {
+        let buf = Align::<[u8; 8], AU64>::default();
+        // `buf.t` should be aligned to 8, so this should always succeed.
+        b.iter(|| test::black_box(Ref::<_, AU64>::from_bytes(&buf.t[..]).unwrap()));
+    }
+
+    #[bench]
+    fn bench_into_ref_sized(b: &mut Bencher) {
+        let buf = Align::<[u8; 8], AU64>::default();
+        b.iter(|| {
+            let r = Ref::<_, AU64>::from_bytes(&buf.t[..]).unwrap();
+            test::black_box(Ref::into_ref(r))
+        });
+    }
+
+    #[bench]
+    fn bench_into_mut_sized(b: &mut Bencher) {
+        let mut buf = Align::<[u8; 8], AU64>::default();
+        b.iter(|| {
+            let r = Ref::<_, AU64>::from_bytes(&mut buf.t[..]).unwrap();
+            test::black_box(Ref::into_mut(r));
+        });
+    }
+
+    #[bench]
+    fn bench_deref_sized(b: &mut Bencher) {
+        let buf = Align::<[u8; 8], AU64>::default();
+        let r = Ref::<_, AU64>::from_bytes(&buf.t[..]).unwrap();
+        b.iter(|| test::black_box(r.deref()));
+    }
+
+    #[bench]
+    fn bench_deref_mut_sized(b: &mut Bencher) {
+        let mut buf = Align::<[u8; 8], AU64>::default();
+        b.iter(|| {
+            let mut r = Ref::<_, AU64>::from_bytes(&mut buf.t[..]).unwrap();
+            test::black_box(r.deref_mut());
+        });
+    }
 
     #[test]
     fn test_mut_slice_into_ref() {