performance: faster utf8 validation by using simd instructions (#4347)

Signed-off-by: Robert Kruszewski <[email protected]>

Author: robert3005

Cargo.lock

@@ -167,6 +167,7 @@ rustversion = "1.0"
 serde = "1.0.203"
 serde_json = "1.0.116"
 serde_test = "1.0.176"
+simdutf8 = "0.1.5"
 similar = "2.7.0"
 simplelog = "0.12"
 sketches-ddsketch = "0.3.0"

Cargo.toml

@@ -51,6 +51,7 @@ rstest = { workspace = true, optional = true }
 rstest_reuse = { workspace = true, optional = true }
 rustc-hash = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
+simdutf8 = { workspace = true }
 static_assertions = { workspace = true }
 tabled = { workspace = true, optional = true, default-features = false, features = [
     "std",

java/testfiles/Cargo.lock

@@ -398,7 +398,7 @@ impl VarBinViewArray {
 
         match dtype {
             DType::Utf8(_) => Self::validate_views(views, buffers, validity, |string| {
-                std::str::from_utf8(string).is_ok()
+                simdutf8::basic::from_utf8(string).is_ok()
             })?,
             DType::Binary(_) => Self::validate_views(views, buffers, validity, |_| true)?,
             _ => vortex_bail!("invalid DType {dtype} for `VarBinViewArray`"),

vortex-array/Cargo.toml

@@ -27,6 +27,7 @@ itertools = { workspace = true }
 log = { workspace = true, optional = true }
 memmap2 = { workspace = true, optional = true }
 num-traits = { workspace = true }
+simdutf8 = { workspace = true }
 vortex-error = { workspace = true }
 
 [lints]

vortex-array/src/arrays/varbinview/mod.rs

@@ -3,7 +3,8 @@
 
 use std::fmt::{Debug, Formatter};
 use std::ops::Deref;
-use std::str::Utf8Error;
+
+use vortex_error::{VortexError, vortex_err};
 
 use crate::ByteBuffer;
 
@@ -64,10 +65,15 @@ impl From<&str> for BufferString {
 }
 
 impl TryFrom<ByteBuffer> for BufferString {
-    type Error = Utf8Error;
+    type Error = VortexError;
 
     fn try_from(value: ByteBuffer) -> Result<Self, Self::Error> {
-        let _ = std::str::from_utf8(value.as_ref())?;
+        let _ = simdutf8::basic::from_utf8(value.as_ref()).map_err(|_| {
+            #[allow(clippy::unwrap_used)]
+            // run validation using `compat` package to get more detailed error message
+            let err = simdutf8::compat::from_utf8(value.as_ref()).unwrap_err();
+            vortex_err!("invalid utf-8: {err}")
+        })?;
         Ok(Self(value))
     }
 }

vortex-buffer/Cargo.toml

@@ -157,14 +157,16 @@ fn decode_timestamp_metadata(ext_meta: &ExtMetadata) -> VortexResult<TemporalMet
     let tag = ext_meta.as_ref()[0];
     let time_unit = TimeUnit::try_from(tag)?;
     let tz_len_bytes = &ext_meta.as_ref()[1..3];
-    let tz_len = u16::from_le_bytes(tz_len_bytes.try_into()?);
+    let tz_len = u16::from_le_bytes(tz_len_bytes.try_into()?) as usize;
     if tz_len == 0 {
         return Ok(TemporalMetadata::Timestamp(time_unit, None));
     }
 
     // Attempt to load from len-prefixed bytes
-    let tz_bytes = &ext_meta.as_ref()[3..(3 + (tz_len as usize))];
-    let tz = String::from_utf8_lossy(tz_bytes).to_string();
+    let tz_bytes = &ext_meta.as_ref()[3..][..tz_len];
+    let tz = str::from_utf8(tz_bytes)
+        .map_err(|e| vortex_err!("timezone is not valid utf8 string: {e}"))?
+        .to_string();
     Ok(TemporalMetadata::Timestamp(time_unit, Some(tz)))
 }
 

vortex-buffer/src/string.rs

@@ -33,7 +33,10 @@ pub fn try_from_bound_expression(value: &Expression) -> VortexResult<Option<Expr
         vortex_bail!("no expression class id {:?}", value.as_class_id())
     };
     Ok(Some(match value {
-        ExpressionClass::BoundColumnRef(col_ref) => col(col_ref.name.to_str()?),
+        ExpressionClass::BoundColumnRef(col_ref) => col(col_ref
+            .name
+            .to_str()
+            .map_err(|e| vortex_err!("invalid utf-8: {e}"))?),
         ExpressionClass::BoundConstant(const_) => lit(Scalar::try_from(const_.value)?),
         ExpressionClass::BoundComparison(compare) => {
             let operator: Operator = compare.op.try_into()?;

vortex-dtype/src/datetime/temporal.rs

@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
-use vortex::error::VortexExpect;
+use vortex::error::{VortexUnwrap, vortex_err};
 
 use crate::duckdb::LogicalType;
 use crate::{cpp, wrapper};
@@ -14,7 +14,8 @@ impl ScalarFunction {
             let name_ptr = cpp::duckdb_vx_sfunc_name(self.as_ptr());
             std::ffi::CStr::from_ptr(name_ptr)
                 .to_str()
-                .vortex_expect("invalid utf-8")
+                .map_err(|e| vortex_err!("invalid utf-8: {e}"))
+                .vortex_unwrap()
         }
     }