Use simdutf8

Author: utkarshgupta137

Cargo.lock

@@ -35,6 +35,7 @@ regex-automata = "0.4.14"
 serde = { version = "1.0.228", features = ["derive"] }
 serde_json = "1.0.149"
 serde-wasm-bindgen = "0.6.5"
+simdutf8 = "0.1.5"
 sliceslice = "0.4.3"
 thiserror = "2.0.18"
 wasm-bindgen = { version = "0.2.108", features = ["serde-serialize"] }

Cargo.toml

@@ -30,6 +30,7 @@ memchr.workspace = true
 rand.workspace = true
 regex-automata = { workspace = true, optional = true }
 serde.workspace = true
+simdutf8.workspace = true
 sliceslice.workspace = true
 thiserror.workspace = true
 wildcard.workspace = true

engine/Cargo.toml

@@ -936,7 +936,10 @@ mod tests {
                     LhsValue::Bytes(bytes) => bytes,
                     _ => unreachable!(),
                 };
-                assert_eq!(std::str::from_utf8(&bytes).unwrap(), format!("[{i}][{j}]"));
+                assert_eq!(
+                    simdutf8::basic::from_utf8(&bytes).unwrap(),
+                    format!("[{i}][{j}]")
+                );
             }
 
             let indexes = [FieldIndex::MapEach, FieldIndex::ArrayIndex(i)];
@@ -948,7 +951,10 @@ mod tests {
                     LhsValue::Bytes(bytes) => bytes,
                     _ => unreachable!(),
                 };
-                assert_eq!(std::str::from_utf8(&bytes).unwrap(), format!("[{j}][{i}]"));
+                assert_eq!(
+                    simdutf8::basic::from_utf8(&bytes).unwrap(),
+                    format!("[{j}][{i}]")
+                );
             }
         }
 
@@ -963,7 +969,10 @@ mod tests {
                 LhsValue::Bytes(bytes) => bytes,
                 _ => unreachable!(),
             };
-            assert_eq!(std::str::from_utf8(&bytes).unwrap(), format!("[{i}][{j}]"));
+            assert_eq!(
+                simdutf8::basic::from_utf8(&bytes).unwrap(),
+                format!("[{i}][{j}]")
+            );
             j = (j + 1) % 10;
             i += (j == 0) as u32;
         }

engine/src/ast/index_expr.rs

@@ -254,7 +254,7 @@ impl Serialize for Bytes<'_> {
     where
         S: Serializer,
     {
-        if let Ok(s) = std::str::from_utf8(self) {
+        if let Ok(s) = simdutf8::basic::from_utf8(self) {
             serializer.serialize_str(s)
         } else {
             serializer.serialize_bytes(self)

engine/src/lhs_types/bytes.rs

@@ -339,12 +339,12 @@ impl Serialize for Map<'_> {
         let to_map = self
             .data
             .iter()
-            .all(|(key, _)| std::str::from_utf8(key).is_ok());
+            .all(|(key, _)| simdutf8::basic::from_utf8(key).is_ok());
 
         if to_map {
             let mut map = serializer.serialize_map(Some(self.len()))?;
             for (k, v) in self.data.iter() {
-                map.serialize_entry(std::str::from_utf8(k).unwrap(), v)?;
+                map.serialize_entry(simdutf8::basic::from_utf8(k).unwrap(), v)?;
             }
             map.end()
         } else {

engine/src/lhs_types/map.rs

@@ -4,7 +4,6 @@ use serde::{Serialize, Serializer};
 use std::fmt::{self, Debug, Formatter};
 use std::hash::{Hash, Hasher};
 use std::ops::Deref;
-use std::str;
 
 /// BytesFormat describes the format in which the string was expressed
 #[derive(PartialEq, Eq, Copy, Clone)]
@@ -48,10 +47,12 @@ impl Serialize for BytesExpr {
         S: Serializer,
     {
         match self.format() {
-            BytesFormat::Quoted | BytesFormat::Raw(_) => match std::str::from_utf8(&self.data) {
-                Ok(s) => s.serialize(serializer),
-                Err(_) => self.data.serialize(serializer),
-            },
+            BytesFormat::Quoted | BytesFormat::Raw(_) => {
+                match simdutf8::basic::from_utf8(&self.data) {
+                    Ok(s) => s.serialize(serializer),
+                    Err(_) => self.data.serialize(serializer),
+                }
+            }
             BytesFormat::Byte => self.data.serialize(serializer),
         }
     }
@@ -117,10 +118,12 @@ impl Debug for BytesExpr {
         }
 
         match self.format {
-            BytesFormat::Quoted | BytesFormat::Raw(_) => match std::str::from_utf8(&self.data) {
-                Ok(s) => s.fmt(f),
-                Err(_) => fmt_raw(&self.data, f),
-            },
+            BytesFormat::Quoted | BytesFormat::Raw(_) => {
+                match simdutf8::basic::from_utf8(&self.data) {
+                    Ok(s) => s.fmt(f),
+                    Err(_) => fmt_raw(&self.data, f),
+                }
+            }
             BytesFormat::Byte => fmt_raw(&self.data, f),
         }
     }

engine/src/rhs_types/bytes.rs

@@ -76,10 +76,16 @@ impl<'i> Lex<'i> for FieldIndex {
                     input,
                 )),
             },
-            RhsValue::Bytes(b) => match String::from_utf8(b.into()) {
-                Ok(s) => Ok((FieldIndex::MapKey(s), rest)),
-                Err(_) => Err((LexErrorKind::ExpectedLiteral("expected utf8 string"), input)),
-            },
+            RhsValue::Bytes(b) => {
+                match simdutf8::basic::from_utf8(&b) {
+                    Ok(_) => {
+                        // SAFETY: simdutf8 just validated the bytes as valid UTF-8.
+                        let s = unsafe { String::from_utf8_unchecked(b.into()) };
+                        Ok((FieldIndex::MapKey(s), rest))
+                    }
+                    Err(_) => Err((LexErrorKind::ExpectedLiteral("expected utf8 string"), input)),
+                }
+            }
             _ => unreachable!(),
         }
     }

engine/src/scheme.rs

@@ -785,7 +785,7 @@ impl Serialize for LhsValue<'_> {
         match self {
             LhsValue::Ip(ip) => ip.serialize(serializer),
             LhsValue::Bytes(bytes) => {
-                if let Ok(s) = std::str::from_utf8(bytes) {
+                if let Ok(s) = simdutf8::basic::from_utf8(bytes) {
                     serializer.serialize_str(s)
                 } else {
                     serializer.serialize_bytes(bytes)