Merge branch 'main' of github.com:rusticstuff/simdutf8

hkratz · hkratz · commit 84b79cf89a18 · 2021-04-26T10:32:34.000+02:00
diff --git a/src/implementation/macros.rs b/src/implementation/macros.rs
@@ -37,23 +37,34 @@ macro_rules! static_cast_i8 {
     };
 }
 
+macro_rules! check_utf8 {
+    ($feat:expr, $t:ident) => {
+        #[target_feature(enable = $feat)]
+        #[inline]
+        unsafe fn check_utf8(&self, previous: &mut Utf8CheckingState<$t>) {
+            if likely!(self.is_ascii()) {
+                previous.error =
+                    Utf8CheckingState::<$t>::check_eof(previous.error, previous.incomplete)
+            } else {
+                self.check_block(previous);
+            }
+        }
+    };
+}
+
 /// check_bytes() strategy
 macro_rules! check_bytes {
     ($feat:expr, $t:ident) => {
         #[target_feature(enable = $feat)]
         #[inline]
         unsafe fn check_bytes(current: $t, previous: &mut Utf8CheckingState<$t>) {
-            if likely!(Self::is_ascii(current)) {
-                previous.error = Self::check_eof(previous.error, previous.incomplete)
-            } else {
-                let prev1 = Self::prev1(current, previous.prev);
-                let sc = Self::check_special_cases(current, prev1);
-                previous.error = Self::or(
-                    previous.error,
-                    Self::check_multibyte_lengths(current, previous.prev, sc),
-                );
-                previous.incomplete = Self::is_incomplete(current);
-            }
+            let prev1 = Self::prev1(current, previous.prev);
+            let sc = Self::check_special_cases(current, prev1);
+            previous.error = Self::or(
+                previous.error,
+                Self::check_multibyte_lengths(current, previous.prev, sc),
+            );
+            previous.incomplete = Self::is_incomplete(current);
             previous.prev = current
         }
     };
@@ -79,12 +90,11 @@ macro_rules! validate_utf8_basic_simd {
             const SIMDINPUT_LENGTH: usize = 64;
             let len = input.len();
             let mut state = SimdInput::new_utf8_checking_state();
-            let lenminus64: usize = if len < 64 { 0 } else { len as usize - 64 };
             let mut idx: usize = 0;
             let mut tmpbuf = $buf2type::new();
 
             let align: usize = core::mem::align_of::<$buf2type>();
-            if lenminus64 >= 4096 {
+            if len >= 4096 {
                 let off = (input.as_ptr() as usize) % align;
                 if off != 0 {
                     let to_copy = align - off;
@@ -98,7 +108,10 @@ macro_rules! validate_utf8_basic_simd {
                     idx += to_copy;
                 }
             }
-            while idx < lenminus64 {
+
+            let rem = len - idx;
+            let iter_lim = idx + (rem - (rem % 64));
+            while idx < iter_lim {
                 let input = SimdInput::new(input.get_unchecked(idx as usize..));
                 input.check_utf8(&mut state);
                 idx += SIMDINPUT_LENGTH;
@@ -151,12 +164,11 @@ macro_rules! validate_utf8_compat_simd {
             const SIMDINPUT_LENGTH: usize = 64;
             let len = input.len();
             let mut state = SimdInput::new_utf8_checking_state();
-            let lenminus64: usize = if len < 64 { 0 } else { len as usize - 64 };
             let mut idx: usize = 0;
             let mut tmpbuf = $buf2type::new();
 
             let align: usize = core::mem::align_of::<$buf2type>();
-            if lenminus64 >= 4096 {
+            if len >= 4096 {
                 let off = (input.as_ptr() as usize) % align;
                 if off != 0 {
                     let to_copy = align - off;
@@ -174,7 +186,9 @@ macro_rules! validate_utf8_compat_simd {
                 }
             }
 
-            while idx < lenminus64 {
+            let rem = len - idx;
+            let iter_lim = idx + (rem - (rem % 64));
+            while idx < iter_lim {
                 let simd_input = SimdInput::new(input.get_unchecked(idx as usize..));
                 simd_input.check_utf8(&mut state);
                 if SimdInput::check_utf8_errors(&state) {
diff --git a/src/implementation/x86/avx2.rs b/src/implementation/x86/avx2.rs
@@ -37,12 +37,6 @@ impl Utf8CheckingState<__m256i> {
         _mm256_or_si256(a, b)
     }
 
-    #[target_feature(enable = "avx2")]
-    #[inline]
-    unsafe fn is_ascii(input: __m256i) -> bool {
-        _mm256_movemask_epi8(input) == 0
-    }
-
     #[target_feature(enable = "avx2")]
     #[inline]
     unsafe fn check_eof(error: __m256i, incomplete: __m256i) -> __m256i {
@@ -272,8 +266,6 @@ impl Utf8CheckingState<__m256i> {
         _mm256_testz_si256(error, error) != 1
     }
 
-    #[target_feature(enable = "avx2")]
-    #[inline]
     check_bytes!("avx2", __m256i);
 }
 
@@ -302,11 +294,18 @@ impl SimdInput {
 
     #[target_feature(enable = "avx2")]
     #[inline]
-    unsafe fn check_utf8(&self, state: &mut Utf8CheckingState<__m256i>) {
+    unsafe fn check_block(&self, state: &mut Utf8CheckingState<__m256i>) {
         Utf8CheckingState::<__m256i>::check_bytes(self.v0, state);
         Utf8CheckingState::<__m256i>::check_bytes(self.v1, state);
     }
 
+    #[target_feature(enable = "avx2")]
+    #[inline]
+    unsafe fn is_ascii(&self) -> bool {
+        let res = _mm256_or_si256(self.v0, self.v1);
+        _mm256_movemask_epi8(res) == 0
+    }
+
     #[target_feature(enable = "avx2")]
     #[inline]
     unsafe fn check_eof(state: &mut Utf8CheckingState<__m256i>) {
@@ -318,6 +317,8 @@ impl SimdInput {
     unsafe fn check_utf8_errors(state: &Utf8CheckingState<__m256i>) -> bool {
         Utf8CheckingState::<__m256i>::has_error(state.error)
     }
+
+    check_utf8!("avx2", __m256i);
 }
 
 use crate::implementation::Temp2x64A32;
diff --git a/src/implementation/x86/mod.rs b/src/implementation/x86/mod.rs
@@ -24,6 +24,10 @@ pub(crate) unsafe fn validate_utf8_basic(
         (fun)(input)
     }
 
+    if input.len() < 64 {
+        return super::validate_utf8_basic_fallback(input);
+    }
+
     let fun = FN.load(Ordering::Relaxed);
     mem::transmute::<FnRaw, super::ValidateUtf8Fn>(fun)(input)
 }
@@ -46,6 +50,10 @@ fn get_fastest_available_implementation_basic() -> super::ValidateUtf8Fn {
 pub(crate) unsafe fn validate_utf8_basic(
     input: &[u8],
 ) -> core::result::Result<(), crate::basic::Utf8Error> {
+    if input.len() < 64 {
+        return super::validate_utf8_basic_fallback(input);
+    }
+
     avx2::validate_utf8_basic(input)
 }
 
@@ -57,6 +65,10 @@ pub(crate) unsafe fn validate_utf8_basic(
 pub(crate) unsafe fn validate_utf8_basic(
     input: &[u8],
 ) -> core::result::Result<(), crate::basic::Utf8Error> {
+    if input.len() < 64 {
+        return super::validate_utf8_basic_fallback(input);
+    }
+
     sse42::validate_utf8_basic(input)
 }
 
@@ -88,6 +100,10 @@ pub(crate) unsafe fn validate_utf8_compat(
         (fun)(input)
     }
 
+    if input.len() < 64 {
+        return super::validate_utf8_compat_fallback(input);
+    }
+
     let fun = FN.load(Ordering::Relaxed);
     mem::transmute::<FnRaw, super::ValidateUtf8CompatFn>(fun)(input)
 }
@@ -110,6 +126,10 @@ fn get_fastest_available_implementation_compat() -> super::ValidateUtf8CompatFn
 pub(crate) unsafe fn validate_utf8_compat(
     input: &[u8],
 ) -> core::result::Result<(), crate::compat::Utf8Error> {
+    if input.len() < 64 {
+        return super::validate_utf8_compat_fallback(input);
+    }
+
     avx2::validate_utf8_compat(input)
 }
 
@@ -121,6 +141,10 @@ pub(crate) unsafe fn validate_utf8_compat(
 pub(crate) unsafe fn validate_utf8_compat(
     input: &[u8],
 ) -> core::result::Result<(), crate::compat::Utf8Error> {
+    if input.len() < 64 {
+        return super::validate_utf8_compat_fallback(input);
+    }
+
     sse42::validate_utf8_compat(input)
 }
 
diff --git a/src/implementation/x86/sse42.rs b/src/implementation/x86/sse42.rs
@@ -36,12 +36,6 @@ impl Utf8CheckingState<__m128i> {
         _mm_or_si128(a, b)
     }
 
-    #[target_feature(enable = "sse4.2")]
-    #[inline]
-    unsafe fn is_ascii(input: __m128i) -> bool {
-        _mm_movemask_epi8(input) == 0
-    }
-
     #[target_feature(enable = "sse4.2")]
     #[inline]
     unsafe fn check_eof(error: __m128i, incomplete: __m128i) -> __m128i {
@@ -205,8 +199,6 @@ impl Utf8CheckingState<__m128i> {
         _mm_testz_si128(error, error) != 1
     }
 
-    #[target_feature(enable = "sse4.2")]
-    #[inline]
     check_bytes!("sse4.2", __m128i);
 }
 
@@ -239,13 +231,22 @@ impl SimdInput {
 
     #[target_feature(enable = "sse4.2")]
     #[inline]
-    unsafe fn check_utf8(&self, state: &mut Utf8CheckingState<__m128i>) {
+    unsafe fn check_block(&self, state: &mut Utf8CheckingState<__m128i>) {
         Utf8CheckingState::<__m128i>::check_bytes(self.v0, state);
         Utf8CheckingState::<__m128i>::check_bytes(self.v1, state);
         Utf8CheckingState::<__m128i>::check_bytes(self.v2, state);
         Utf8CheckingState::<__m128i>::check_bytes(self.v3, state);
     }
 
+    #[target_feature(enable = "sse4.2")]
+    #[inline]
+    unsafe fn is_ascii(&self) -> bool {
+        let r1 = _mm_or_si128(self.v0, self.v1);
+        let r2 = _mm_or_si128(self.v2, self.v3);
+        let r = _mm_or_si128(r1, r2);
+        _mm_movemask_epi8(r) == 0
+    }
+
     #[target_feature(enable = "sse4.2")]
     #[inline]
     unsafe fn check_eof(state: &mut Utf8CheckingState<__m128i>) {
@@ -257,6 +258,8 @@ impl SimdInput {
     unsafe fn check_utf8_errors(state: &Utf8CheckingState<__m128i>) -> bool {
         Utf8CheckingState::<__m128i>::has_error(state.error)
     }
+
+    check_utf8!("sse4.2", __m128i);
 }
 
 use crate::implementation::Temp2x64A16;
