Merge branch 'master' into sync-upstream-2023-06-07

Author: calebzulawski

crates/core_simd/src/cast.rs

@@ -1,55 +1,51 @@
 use crate::simd::SimdElement;
 
+mod sealed {
+    /// Cast vector elements to other types.
+    ///
+    /// # Safety
+    /// Implementing this trait asserts that the type is a valid vector element for the `simd_cast`
+    /// or `simd_as` intrinsics.
+    pub unsafe trait Sealed {}
+}
+use sealed::Sealed;
+
 /// Supporting trait for `Simd::cast`.  Typically doesn't need to be used directly.
-///
-/// # Safety
-/// Implementing this trait asserts that the type is a valid vector element for the `simd_cast` or
-/// `simd_as` intrinsics.
-pub unsafe trait SimdCast: SimdElement {}
+pub trait SimdCast: Sealed + SimdElement {}
 
 // Safety: primitive number types can be cast to other primitive number types
-unsafe impl SimdCast for i8 {}
+unsafe impl Sealed for i8 {}
+impl SimdCast for i8 {}
 // Safety: primitive number types can be cast to other primitive number types
-unsafe impl SimdCast for i16 {}
+unsafe impl Sealed for i16 {}
+impl SimdCast for i16 {}
 // Safety: primitive number types can be cast to other primitive number types
-unsafe impl SimdCast for i32 {}
+unsafe impl Sealed for i32 {}
+impl SimdCast for i32 {}
 // Safety: primitive number types can be cast to other primitive number types
-unsafe impl SimdCast for i64 {}
+unsafe impl Sealed for i64 {}
+impl SimdCast for i64 {}
 // Safety: primitive number types can be cast to other primitive number types
-unsafe impl SimdCast for isize {}
+unsafe impl Sealed for isize {}
+impl SimdCast for isize {}
 // Safety: primitive number types can be cast to other primitive number types
-unsafe impl SimdCast for u8 {}
+unsafe impl Sealed for u8 {}
+impl SimdCast for u8 {}
 // Safety: primitive number types can be cast to other primitive number types
-unsafe impl SimdCast for u16 {}
+unsafe impl Sealed for u16 {}
+impl SimdCast for u16 {}
 // Safety: primitive number types can be cast to other primitive number types
-unsafe impl SimdCast for u32 {}
+unsafe impl Sealed for u32 {}
+impl SimdCast for u32 {}
 // Safety: primitive number types can be cast to other primitive number types
-unsafe impl SimdCast for u64 {}
+unsafe impl Sealed for u64 {}
+impl SimdCast for u64 {}
 // Safety: primitive number types can be cast to other primitive number types
-unsafe impl SimdCast for usize {}
+unsafe impl Sealed for usize {}
+impl SimdCast for usize {}
 // Safety: primitive number types can be cast to other primitive number types
-unsafe impl SimdCast for f32 {}
+unsafe impl Sealed for f32 {}
+impl SimdCast for f32 {}
 // Safety: primitive number types can be cast to other primitive number types
-unsafe impl SimdCast for f64 {}
-
-/// Supporting trait for `Simd::cast_ptr`.  Typically doesn't need to be used directly.
-///
-/// # Safety
-/// Implementing this trait asserts that the type is a valid vector element for the `simd_cast_ptr`
-/// intrinsic.
-pub unsafe trait SimdCastPtr<T> {}
-
-// Safety: pointers can be cast to other pointer types
-unsafe impl<T, U> SimdCastPtr<T> for *const U
-where
-    U: core::ptr::Pointee,
-    T: core::ptr::Pointee<Metadata = U::Metadata>,
-{
-}
-// Safety: pointers can be cast to other pointer types
-unsafe impl<T, U> SimdCastPtr<T> for *mut U
-where
-    U: core::ptr::Pointee,
-    T: core::ptr::Pointee<Metadata = U::Metadata>,
-{
-}
+unsafe impl Sealed for f64 {}
+impl SimdCast for f64 {}

crates/core_simd/src/elements/const_ptr.rs

@@ -1,5 +1,5 @@
 use super::sealed::Sealed;
-use crate::simd::{intrinsics, LaneCount, Mask, Simd, SimdPartialEq, SupportedLaneCount};
+use crate::simd::{intrinsics, LaneCount, Mask, Simd, SimdPartialEq, SimdUint, SupportedLaneCount};
 
 /// Operations on SIMD vectors of constant pointers.
 pub trait SimdConstPtr: Copy + Sealed {
@@ -9,6 +9,9 @@ pub trait SimdConstPtr: Copy + Sealed {
     /// Vector of `isize` with the same number of lanes.
     type Isize;
 
+    /// Vector of const pointers with the same number of lanes.
+    type CastPtr<T>;
+
     /// Vector of mutable pointers to the same type.
     type MutPtr;
 
@@ -18,6 +21,11 @@ pub trait SimdConstPtr: Copy + Sealed {
     /// Returns `true` for each lane that is null.
     fn is_null(self) -> Self::Mask;
 
+    /// Casts to a pointer of another type.
+    ///
+    /// Equivalent to calling [`pointer::cast`] on each lane.
+    fn cast<T>(self) -> Self::CastPtr<T>;
+
     /// Changes constness without changing the type.
     ///
     /// Equivalent to calling [`pointer::cast_mut`] on each lane.
@@ -78,6 +86,7 @@ where
 {
     type Usize = Simd<usize, LANES>;
     type Isize = Simd<isize, LANES>;
+    type CastPtr<U> = Simd<*const U, LANES>;
     type MutPtr = Simd<*mut T, LANES>;
     type Mask = Mask<isize, LANES>;
 
@@ -86,9 +95,22 @@ where
         Simd::splat(core::ptr::null()).simd_eq(self)
     }
 
+    #[inline]
+    fn cast<U>(self) -> Self::CastPtr<U> {
+        // SimdElement currently requires zero-sized metadata, so this should never fail.
+        // If this ever changes, `simd_cast_ptr` should produce a post-mono error.
+        use core::{mem::size_of, ptr::Pointee};
+        assert_eq!(size_of::<<T as Pointee>::Metadata>(), 0);
+        assert_eq!(size_of::<<U as Pointee>::Metadata>(), 0);
+
+        // Safety: pointers can be cast
+        unsafe { intrinsics::simd_cast_ptr(self) }
+    }
+
     #[inline]
     fn cast_mut(self) -> Self::MutPtr {
-        self.cast_ptr()
+        // Safety: pointers can be cast
+        unsafe { intrinsics::simd_cast_ptr(self) }
     }
 
     #[inline]
@@ -106,9 +128,9 @@ where
         // In the mean-time, this operation is defined to be "as if" it was
         // a wrapping_offset, so we can emulate it as such. This should properly
         // restore pointer provenance even under today's compiler.
-        self.cast_ptr::<*const u8>()
+        self.cast::<u8>()
             .wrapping_offset(addr.cast::<isize>() - self.addr().cast::<isize>())
-            .cast_ptr()
+            .cast()
     }
 
     #[inline]

crates/core_simd/src/elements/float.rs

@@ -1,6 +1,6 @@
 use super::sealed::Sealed;
 use crate::simd::{
-    intrinsics, LaneCount, Mask, Simd, SimdElement, SimdPartialEq, SimdPartialOrd,
+    intrinsics, LaneCount, Mask, Simd, SimdCast, SimdElement, SimdPartialEq, SimdPartialOrd,
     SupportedLaneCount,
 };
 
@@ -15,6 +15,51 @@ pub trait SimdFloat: Copy + Sealed {
     /// Bit representation of this SIMD vector type.
     type Bits;
 
+    /// A SIMD vector with a different element type.
+    type Cast<T: SimdElement>;
+
+    /// Performs elementwise conversion of this vector's elements to another SIMD-valid type.
+    ///
+    /// This follows the semantics of Rust's `as` conversion for floats (truncating or saturating
+    /// at the limits) for each element.
+    ///
+    /// # Example
+    /// ```
+    /// # #![feature(portable_simd)]
+    /// # use core::simd::Simd;
+    /// let floats: Simd<f32, 4> = Simd::from_array([1.9, -4.5, f32::INFINITY, f32::NAN]);
+    /// let ints = floats.cast::<i32>();
+    /// assert_eq!(ints, Simd::from_array([1, -4, i32::MAX, 0]));
+    ///
+    /// // Formally equivalent, but `Simd::cast` can optimize better.
+    /// assert_eq!(ints, Simd::from_array(floats.to_array().map(|x| x as i32)));
+    ///
+    /// // The float conversion does not round-trip.
+    /// let floats_again = ints.cast();
+    /// assert_ne!(floats, floats_again);
+    /// assert_eq!(floats_again, Simd::from_array([1.0, -4.0, 2147483647.0, 0.0]));
+    /// ```
+    #[must_use]
+    fn cast<T: SimdCast>(self) -> Self::Cast<T>;
+
+    /// Rounds toward zero and converts to the same-width integer type, assuming that
+    /// the value is finite and fits in that type.
+    ///
+    /// # Safety
+    /// The value must:
+    ///
+    /// * Not be NaN
+    /// * Not be infinite
+    /// * Be representable in the return type, after truncating off its fractional part
+    ///
+    /// If these requirements are infeasible or costly, consider using the safe function [cast],
+    /// which saturates on conversion.
+    ///
+    /// [cast]: Simd::cast
+    unsafe fn to_int_unchecked<I: SimdCast>(self) -> Self::Cast<I>
+    where
+        Self::Scalar: core::convert::FloatToInt<I>;
+
     /// Raw transmutation to an unsigned integer vector type with the
     /// same size and number of lanes.
     #[must_use = "method returns a new vector and does not mutate the original value"]
@@ -206,6 +251,24 @@ macro_rules! impl_trait {
             type Mask = Mask<<$mask_ty as SimdElement>::Mask, LANES>;
             type Scalar = $ty;
             type Bits = Simd<$bits_ty, LANES>;
+            type Cast<T: SimdElement> = Simd<T, LANES>;
+
+            #[inline]
+            fn cast<T: SimdCast>(self) -> Self::Cast<T>
+            {
+                // Safety: supported types are guaranteed by SimdCast
+                unsafe { intrinsics::simd_as(self) }
+            }
+
+            #[inline]
+            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
+            unsafe fn to_int_unchecked<I: SimdCast>(self) -> Self::Cast<I>
+            where
+                Self::Scalar: core::convert::FloatToInt<I>,
+            {
+                // Safety: supported types are guaranteed by SimdCast, the caller is responsible for the extra invariants
+                unsafe { intrinsics::simd_cast(self) }
+            }
 
             #[inline]
             fn to_bits(self) -> Simd<$bits_ty, LANES> {

crates/core_simd/src/elements/int.rs

@@ -1,6 +1,6 @@
 use super::sealed::Sealed;
 use crate::simd::{
-    intrinsics, LaneCount, Mask, Simd, SimdElement, SimdPartialOrd, SupportedLaneCount,
+    intrinsics, LaneCount, Mask, Simd, SimdCast, SimdElement, SimdPartialOrd, SupportedLaneCount,
 };
 
 /// Operations on SIMD vectors of signed integers.
@@ -11,6 +11,16 @@ pub trait SimdInt: Copy + Sealed {
     /// Scalar type contained by this SIMD vector type.
     type Scalar;
 
+    /// A SIMD vector with a different element type.
+    type Cast<T: SimdElement>;
+
+    /// Performs elementwise conversion of this vector's elements to another SIMD-valid type.
+    ///
+    /// This follows the semantics of Rust's `as` conversion for casting integers (wrapping to
+    /// other integer types, and saturating to float types).
+    #[must_use]
+    fn cast<T: SimdCast>(self) -> Self::Cast<T>;
+
     /// Lanewise saturating add.
     ///
     /// # Examples
@@ -198,6 +208,13 @@ macro_rules! impl_trait {
         {
             type Mask = Mask<<$ty as SimdElement>::Mask, LANES>;
             type Scalar = $ty;
+            type Cast<T: SimdElement> = Simd<T, LANES>;
+
+            #[inline]
+            fn cast<T: SimdCast>(self) -> Self::Cast<T> {
+                // Safety: supported types are guaranteed by SimdCast
+                unsafe { intrinsics::simd_as(self) }
+            }
 
             #[inline]
             fn saturating_add(self, second: Self) -> Self {

crates/core_simd/src/elements/mut_ptr.rs

@@ -1,5 +1,5 @@
 use super::sealed::Sealed;
-use crate::simd::{intrinsics, LaneCount, Mask, Simd, SimdPartialEq, SupportedLaneCount};
+use crate::simd::{intrinsics, LaneCount, Mask, Simd, SimdPartialEq, SimdUint, SupportedLaneCount};
 
 /// Operations on SIMD vectors of mutable pointers.
 pub trait SimdMutPtr: Copy + Sealed {
@@ -9,6 +9,9 @@ pub trait SimdMutPtr: Copy + Sealed {
     /// Vector of `isize` with the same number of lanes.
     type Isize;
 
+    /// Vector of const pointers with the same number of lanes.
+    type CastPtr<T>;
+
     /// Vector of constant pointers to the same type.
     type ConstPtr;
 
@@ -18,6 +21,11 @@ pub trait SimdMutPtr: Copy + Sealed {
     /// Returns `true` for each lane that is null.
     fn is_null(self) -> Self::Mask;
 
+    /// Casts to a pointer of another type.
+    ///
+    /// Equivalent to calling [`pointer::cast`] on each lane.
+    fn cast<T>(self) -> Self::CastPtr<T>;
+
     /// Changes constness without changing the type.
     ///
     /// Equivalent to calling [`pointer::cast_const`] on each lane.
@@ -73,6 +81,7 @@ where
 {
     type Usize = Simd<usize, LANES>;
     type Isize = Simd<isize, LANES>;
+    type CastPtr<U> = Simd<*mut U, LANES>;
     type ConstPtr = Simd<*const T, LANES>;
     type Mask = Mask<isize, LANES>;
 
@@ -81,9 +90,22 @@ where
         Simd::splat(core::ptr::null_mut()).simd_eq(self)
     }
 
+    #[inline]
+    fn cast<U>(self) -> Self::CastPtr<U> {
+        // SimdElement currently requires zero-sized metadata, so this should never fail.
+        // If this ever changes, `simd_cast_ptr` should produce a post-mono error.
+        use core::{mem::size_of, ptr::Pointee};
+        assert_eq!(size_of::<<T as Pointee>::Metadata>(), 0);
+        assert_eq!(size_of::<<U as Pointee>::Metadata>(), 0);
+
+        // Safety: pointers can be cast
+        unsafe { intrinsics::simd_cast_ptr(self) }
+    }
+
     #[inline]
     fn cast_const(self) -> Self::ConstPtr {
-        self.cast_ptr()
+        // Safety: pointers can be cast
+        unsafe { intrinsics::simd_cast_ptr(self) }
     }
 
     #[inline]
@@ -101,9 +123,9 @@ where
         // In the mean-time, this operation is defined to be "as if" it was
         // a wrapping_offset, so we can emulate it as such. This should properly
         // restore pointer provenance even under today's compiler.
-        self.cast_ptr::<*mut u8>()
+        self.cast::<u8>()
             .wrapping_offset(addr.cast::<isize>() - self.addr().cast::<isize>())
-            .cast_ptr()
+            .cast()
     }
 
     #[inline]

crates/core_simd/src/elements/uint.rs

@@ -1,11 +1,21 @@
 use super::sealed::Sealed;
-use crate::simd::{intrinsics, LaneCount, Simd, SupportedLaneCount};
+use crate::simd::{intrinsics, LaneCount, Simd, SimdCast, SimdElement, SupportedLaneCount};
 
 /// Operations on SIMD vectors of unsigned integers.
 pub trait SimdUint: Copy + Sealed {
     /// Scalar type contained by this SIMD vector type.
     type Scalar;
 
+    /// A SIMD vector with a different element type.
+    type Cast<T: SimdElement>;
+
+    /// Performs elementwise conversion of this vector's elements to another SIMD-valid type.
+    ///
+    /// This follows the semantics of Rust's `as` conversion for casting integers (wrapping to
+    /// other integer types, and saturating to float types).
+    #[must_use]
+    fn cast<T: SimdCast>(self) -> Self::Cast<T>;
+
     /// Lanewise saturating add.
     ///
     /// # Examples
@@ -77,6 +87,13 @@ macro_rules! impl_trait {
             LaneCount<LANES>: SupportedLaneCount,
         {
             type Scalar = $ty;
+            type Cast<T: SimdElement> = Simd<T, LANES>;
+
+            #[inline]
+            fn cast<T: SimdCast>(self) -> Self::Cast<T> {
+                // Safety: supported types are guaranteed by SimdCast
+                unsafe { intrinsics::simd_as(self) }
+            }
 
             #[inline]
             fn saturating_add(self, second: Self) -> Self {