JS TextDecoder: safe and decodeBOM options

Signed-off-by: Didier Wenzek <[email protected]>

Author: didier-wenzek

crates/extensions/tedge_flows/src/js_lib/text_decoder.rs

@@ -1,14 +1,20 @@
 use rquickjs::class::Trace;
+use rquickjs::prelude::Opt;
 use rquickjs::Class;
 use rquickjs::Ctx;
 use rquickjs::Exception;
 use rquickjs::JsLifetime;
+use rquickjs::Object;
 use rquickjs::Result;
 use rquickjs::TypedArray;
+use std::borrow::Cow;
 
-#[derive(Clone, Trace, JsLifetime)]
+#[derive(Clone, Default, Trace, JsLifetime)]
 #[rquickjs::class(frozen)]
-struct TextDecoder {}
+struct TextDecoder {
+    fatal: bool,
+    ignore_bom: bool,
+}
 
 pub fn init(ctx: &Ctx<'_>) {
     let globals = ctx.globals();
@@ -18,22 +24,59 @@ pub fn init(ctx: &Ctx<'_>) {
 #[rquickjs::methods]
 impl<'js> TextDecoder {
     #[qjs(constructor)]
-    fn new() -> TextDecoder {
-        TextDecoder {}
+    fn new(ctx: Ctx<'js>, label: Opt<String>, options: Opt<Object<'js>>) -> Result<TextDecoder> {
+        if let Some(label) = label.into_inner() {
+            if label != "utf-8" && label != "utf8" {
+                return Err(Exception::throw_message(
+                    &ctx,
+                    "TextDecoder only supports utf-8",
+                ));
+            }
+        }
+        let decoder = options.into_inner().map(|options| {
+            let fatal = options.get("fatal").ok().flatten().unwrap_or(false);
+            let ignore_bom = options.get("ignoreBOM").ok().flatten().unwrap_or(false);
+            TextDecoder { fatal, ignore_bom }
+        });
+
+        Ok(decoder.unwrap_or_default())
     }
 
     #[qjs(get)]
     fn encoding(&self) -> &str {
         "utf-8"
     }
 
+    #[qjs(get)]
+    fn fatal(&self) -> bool {
+        self.fatal
+    }
+
+    #[qjs(get, rename = "ignoreBOM")]
+    fn ignore_bom(&self) -> bool {
+        self.ignore_bom
+    }
+
     pub fn decode(&self, ctx: Ctx<'js>, bytes: TypedArray<'js, u8>) -> Result<String> {
-        let bytes = bytes
+        let mut bytes = bytes
             .as_bytes()
             .ok_or(Exception::throw_message(&ctx, "ArrayBuffer is detached"))?;
-        let text = std::str::from_utf8(bytes)
-            .map_err(|err| Exception::throw_message(&ctx, &err.to_string()))?;
 
-        Ok(text.to_owned())
+        if !self.ignore_bom && bytes.get(..3) == Some(&[0xEF, 0xBB, 0xBF]) {
+            bytes = &bytes[3..];
+        }
+
+        let text = if self.fatal {
+            std::str::from_utf8(bytes)
+                .map_err(|err| Exception::throw_message(&ctx, &err.to_string()))?
+                .to_string()
+        } else {
+            match String::from_utf8_lossy(bytes) {
+                Cow::Owned(text) => text,
+                Cow::Borrowed(text) => text.to_owned(),
+            }
+        };
+
+        Ok(text)
     }
 }

crates/extensions/tedge_flows/src/js_script.rs

@@ -396,6 +396,30 @@ export async function onMessage(message, config) {
         );
     }
 
+    #[tokio::test]
+    async fn decode_utf8_with_bom_and_invalid_chars() {
+        let js = r#"
+export async function onMessage(message, config) {
+    const utf8decoder = new TextDecoder();
+    const encodedText = message.raw_payload;
+    const decodedText = utf8decoder.decode(encodedText);
+    return [{topic:"decoded", payload: decodedText}];
+}
+        "#;
+        let (runtime, script) = runtime_with(js).await;
+
+        let utf8_with_bom_and_invalid_chars = b"\xEF\xBB\xBFHello \xF0\x90\x80World";
+        let input = Message::new_binary("encoded", utf8_with_bom_and_invalid_chars);
+        let output = Message::new("decoded", "Hello �World");
+        assert_eq!(
+            script
+                .on_message(&runtime, &DateTime::now(), &input)
+                .await
+                .unwrap(),
+            vec![output]
+        );
+    }
+
     #[tokio::test]
     async fn using_standard_built_in_objects() {
         let js = r#"