Support endianness for non-utf8 segments on js

GearsDatapacks · lpil · commit 3016aaabd2bb · 2025-05-12T11:40:27.000+01:00
diff --git a/compiler-core/src/javascript/expression.rs b/compiler-core/src/javascript/expression.rs
@@ -411,27 +411,67 @@ impl<'module, 'a> Generator<'module, 'a> {
                 }
 
                 // UTF16 strings
-                [Opt::Utf16 { .. }] => {
+                [Opt::Utf16 { .. }]
+                | [Opt::Utf16 { .. }, Opt::Big { .. }]
+                | [Opt::Big { .. }, Opt::Utf16 { .. }] => {
                     self.tracker.string_utf16_bit_array_segment_used = true;
-                    Ok(docvec!["stringToUtf16(", value, ")"])
+                    let is_big = "true".to_doc();
+                    Ok(docvec!["stringToUtf16(", value, ", ", is_big, ")"])
+                }
+
+                [Opt::Utf16 { .. }, Opt::Little { .. }]
+                | [Opt::Little { .. }, Opt::Utf16 { .. }] => {
+                    self.tracker.string_utf16_bit_array_segment_used = true;
+                    let is_big = "false".to_doc();
+                    Ok(docvec!["stringToUtf16(", value, ", ", is_big, ")"])
                 }
 
                 // UTF16 codepoints
-                [Opt::Utf16Codepoint { .. }] => {
+                [Opt::Utf16Codepoint { .. }]
+                | [Opt::Utf16Codepoint { .. }, Opt::Big { .. }]
+                | [Opt::Big { .. }, Opt::Utf16Codepoint { .. }] => {
+                    self.tracker.codepoint_utf16_bit_array_segment_used = true;
+                    let is_big = "true".to_doc();
+                    Ok(docvec!["codepointToUtf16(", value, ", ", is_big, ")"])
+                }
+
+                [Opt::Utf16Codepoint { .. }, Opt::Little { .. }]
+                | [Opt::Little { .. }, Opt::Utf16Codepoint { .. }] => {
                     self.tracker.codepoint_utf16_bit_array_segment_used = true;
-                    Ok(docvec!["codepointToUtf16(", value, ")"])
+                    let is_big = "false".to_doc();
+                    Ok(docvec!["codepointToUtf16(", value, ", ", is_big, ")"])
                 }
 
                 // UTF32 strings
-                [Opt::Utf32 { .. }] => {
+                [Opt::Utf32 { .. }]
+                | [Opt::Utf32 { .. }, Opt::Big { .. }]
+                | [Opt::Big { .. }, Opt::Utf32 { .. }] => {
                     self.tracker.string_utf32_bit_array_segment_used = true;
-                    Ok(docvec!["stringToUtf32(", value, ")"])
+                    let is_big = "true".to_doc();
+                    Ok(docvec!["stringToUtf32(", value, ", ", is_big, ")"])
+                }
+
+                [Opt::Utf32 { .. }, Opt::Little { .. }]
+                | [Opt::Little { .. }, Opt::Utf32 { .. }] => {
+                    self.tracker.string_utf32_bit_array_segment_used = true;
+                    let is_big = "false".to_doc();
+                    Ok(docvec!["stringToUtf32(", value, ", ", is_big, ")"])
                 }
 
                 // UTF32 codepoints
-                [Opt::Utf32Codepoint { .. }] => {
+                [Opt::Utf32Codepoint { .. }]
+                | [Opt::Utf32Codepoint { .. }, Opt::Big { .. }]
+                | [Opt::Big { .. }, Opt::Utf32Codepoint { .. }] => {
+                    self.tracker.codepoint_utf32_bit_array_segment_used = true;
+                    let is_big = "true".to_doc();
+                    Ok(docvec!["codepointToUtf32(", value, ", ", is_big, ")"])
+                }
+
+                [Opt::Utf32Codepoint { .. }, Opt::Little { .. }]
+                | [Opt::Little { .. }, Opt::Utf32Codepoint { .. }] => {
                     self.tracker.codepoint_utf32_bit_array_segment_used = true;
-                    Ok(docvec!["codepointToUtf32(", value, ")"])
+                    let is_big = "false".to_doc();
+                    Ok(docvec!["codepointToUtf32(", value, ", ", is_big, ")"])
                 }
 
                 // Bit arrays
diff --git a/compiler-core/src/strings.rs b/compiler-core/src/strings.rs
@@ -1,6 +1,8 @@
 use ecow::EcoString;
 use itertools::Itertools;
 
+use crate::ast::Endianness;
+
 /// Converts any escape sequences from the given string to their correct
 /// bytewise UTF-8 representation and returns the resulting string.
 pub fn convert_string_escape_chars(str: &EcoString) -> EcoString {
@@ -112,31 +114,42 @@ pub fn to_upper_camel_case(string: &str) -> EcoString {
 }
 
 /// Converts a string into its UTF-16 representation in bytes
-pub fn string_to_utf16_bytes(string: &str) -> Vec<u8> {
+pub fn string_to_utf16_bytes(string: &str, endianness: Endianness) -> Vec<u8> {
     let mut bytes = Vec::with_capacity(string.len() * 2);
 
     for character in string.chars() {
         let mut character_buffer = [0, 0];
 
         _ = character.encode_utf16(&mut character_buffer);
 
-        bytes.extend(character_buffer[0].to_le_bytes());
+        let first_two_bytes = match endianness {
+            Endianness::Big => character_buffer[0].to_be_bytes(),
+            Endianness::Little => character_buffer[0].to_le_bytes(),
+        };
+        bytes.extend(first_two_bytes);
 
         if character_buffer[1] != 0 {
-            bytes.extend(character_buffer[1].to_le_bytes());
+            let next_two_bytes = match endianness {
+                Endianness::Big => character_buffer[1].to_be_bytes(),
+                Endianness::Little => character_buffer[1].to_le_bytes(),
+            };
+            bytes.extend(next_two_bytes);
         }
     }
 
     bytes
 }
 
 /// Converts a string into its UTF-32 representation in bytes
-pub fn string_to_utf32_bytes(string: &str) -> Vec<u8> {
+pub fn string_to_utf32_bytes(string: &str, endianness: Endianness) -> Vec<u8> {
     let mut bytes = Vec::with_capacity(string.len() * 4);
 
     for character in string.chars() {
-        let u32 = character as u32;
-        bytes.extend(u32.to_le_bytes());
+        let character_bytes = match endianness {
+            Endianness::Big => (character as u32).to_be_bytes(),
+            Endianness::Little => (character as u32).to_le_bytes(),
+        };
+        bytes.extend(character_bytes);
     }
 
     bytes
diff --git a/compiler-core/templates/prelude.mjs b/compiler-core/templates/prelude.mjs
@@ -1362,14 +1362,15 @@ export function codepointBits(codepoint) {
  * Returns the UTF-16 bytes for a string.
  *
  * @param {string} string
+ * @param {boolean} isBigEndian
  * @returns {Uint8Array}
 */
-export function stringToUtf16(string) {
+export function stringToUtf16(string, isBigEndian) {
   const buffer = new ArrayBuffer(string.length * 2);
-  const bufferView = new Uint16Array(buffer);
+  const bufferView = new DataView(buffer);
 
   for (let i = 0; i < string.length; i++) {
-    bufferView[i] = string.charCodeAt(i);
+    bufferView.setUint16(i * 2, string.charCodeAt(i), !isBigEndian);
   }
 
   return new Uint8Array(buffer);
@@ -1381,10 +1382,11 @@ export function stringToUtf16(string) {
  * Returns the UTF-16 bytes for a single UTF codepoint.
  *
  * @param {UtfCodepoint} codepoint
+ * @param {boolean} isBigEndian
  * @returns {Uint8Array}
  */
-export function codePointToUtf16(codepoint) {
-  return stringToUtf16(String.fromCodePoint(codepoint.value));
+export function codePointToUtf16(codepoint, isBigEndian) {
+  return stringToUtf16(String.fromCodePoint(codepoint.value), isBigEndian);
 }
 
 /**
@@ -1393,20 +1395,21 @@ export function codePointToUtf16(codepoint) {
  * Returns the UTF-32 bytes for a string.
  *
  * @param {string} string
+ * @param {boolean} isBigEndian
  * @returns {Uint8Array}
 */
-export function stringToUtf32(string) {
+export function stringToUtf32(string, isBigEndian) {
   const buffer = new ArrayBuffer(string.length * 4);
-  const bufferView = new Uint32Array(buffer);
+  const bufferView = new DataView(buffer);
   let length = 0;
 
   for (let i = 0; i < string.length; i++) {
-    const codePoint = string.codePointAt(i);
+    const codepoint = string.codePointAt(i);
 
-    bufferView[length] = codePoint;
+    bufferView.setUint32(length * 4, codepoint, !isBigEndian)
     length++;
 
-    if (codePoint > 0xFFFF) {
+    if (codepoint > 0xFFFF) {
       i++;
     }
   }
@@ -1420,10 +1423,11 @@ export function stringToUtf32(string) {
  * Returns the UTF-32 bytes for a single UTF codepoint.
  *
  * @param {UtfCodepoint} codepoint
+ * @param {boolean} isBigEndian
  * @returns {Uint8Array}
  */
-export function codePointToUtf32(codepoint) {
-  return stringToUtf32(String.fromCodePoint(codepoint.value));
+export function codePointToUtf32(codepoint, isBigEndian) {
+  return stringToUtf32(String.fromCodePoint(codepoint.value), isBigEndian);
 }
 
 export class Result extends CustomType {
