Merge pull request atomvm#491 from fadushin/bs_utf_ops

Add support for bs_(get|skip)_utf(8|16|32)

This PR adds support for the following opcodes:

```
(138) bs_get_utf8
(139) bs_skip_utf8
(140) bs_get_utf16
(141) bs_skip_utf16
(142) bs_get_utf32
(143) bs_skip_utf32
```

These changes are made under both the "Apache 2.0" and the "GNU Lesser General
Public License 2.1 or later" license terms (dual license).

SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later

Author: bettio

CHANGELOG.md

@@ -34,6 +34,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added functions `esp:sleep_enable_ext0_wakeup/2` and `esp:sleep_enable_ext1_wakeup/2.`
 - Added support for FP opcodes 94-102 thus removing the need for `AVM_DISABLE_FP=On` with OTP-22+
 - Added support for stacktraces
+- Added support for `utf-8`, `utf-16`, and `utf-32` bit syntax modifiers (put and match)
+
 
 ### Fixed
 - Fixed issue with formatting integers with io:format() on STM32 platform

src/libAtomVM/bitstring.c

@@ -82,10 +82,31 @@ bool bitstring_insert_any_integer(uint8_t *dst, avm_int_t offset, avm_int64_t va
     return true;
 }
 
+static bool is_invalid_codepoint(int32_t v)
+{
+    return (v < 0) || (v > 0x10FFFF) || (v >= 0xD800 && v <= 0xDFFF);
+}
+
+//
+// UTF-8 encoding
+// https://en.wikipedia.org/wiki/UTF-8
+// +----------+----------+----------+----------+----------+
+// | code pt  |  buf[0]  |  buf[1]  |  buf[2]  |  buf[3]  |
+// +----------+----------+----------+----------+----------+
+// | U+0000   | 0xxxxxxx |          |          |          |
+// +----------+----------+----------+----------+----------+
+// | U+0080   | 110xxxxx | 10xxxxxx |          |          |
+// +----------+----------+----------+----------+----------+
+// | U+0800   | 1110xxxx | 10xxxxxx | 10xxxxxx |          |
+// +----------+----------+----------+----------+----------+
+// | U+10000  | 11110xxx | 10xxxxxx | 10xxxxxx | 10xxxxxx |
+// +----------+----------+----------+----------+----------+
+//
+
 bool bitstring_utf8_encode(avm_int_t c, uint8_t *buf, size_t *out_size)
 {
     size_t sz = 0;
-    if (c < 0 || c > 0x10FFFF) {
+    if (is_invalid_codepoint(c)) {
         return false;
     }
     if (c < 0x80) {
@@ -119,10 +140,67 @@ bool bitstring_utf8_encode(avm_int_t c, uint8_t *buf, size_t *out_size)
     return true;
 }
 
+bool bitstring_utf8_decode(const uint8_t *buf, size_t len, int32_t *c, size_t *out_size)
+{
+    if (len == 0) {
+        return false;
+    } else if (len >= 4 && (buf[0] & 0xF8) == 0xF0 && ((buf[1] & 0xC0) == 0x80) && ((buf[2] & 0xC0) == 0x80) && ((buf[3] & 0xC0) == 0x80)) {
+        int32_t v = 0;
+        v |= (buf[0] & 0x07) << 18;
+        v |= (buf[1] & 0x3F) << 12;
+        v |= (buf[2] & 0x3F) << 6;
+        v |= (buf[3] & 0x3F);
+        // overlong encoding or invalid codepoint
+        if (v <= 0x10000 || v > 0x10FFFF) {
+            return false;
+        }
+        *c = v;
+        *out_size = 4;
+        return true;
+    } else if (len >= 3 && (buf[0] & 0xF0) == 0xE0 && ((buf[1] & 0xC0) == 0x80) && ((buf[2] & 0xC0) == 0x80)) {
+        int32_t v = 0;
+        v |= (buf[0] & 0x0F) << 12;
+        v |= (buf[1] & 0x3F) << 6;
+        v |= (buf[2] & 0x3F);
+        // overlong encoding or surrogate
+        if (v < 0x800 || (v >= 0xD800 && v <= 0xDFFF)) {
+            return false;
+        }
+        *c = v;
+        *out_size = 3;
+        return true;
+    } else if (len >= 2 && (buf[0] & 0xE0) == 0xC0 && ((buf[1] & 0xC0) == 0x80)) {
+        int32_t v = 0;
+        v |= (buf[0] & 0x1F) << 6;
+        v |= (buf[1] & 0x3F);
+        // overlong encoding
+        if (v < 0x80) {
+            return false;
+        }
+        *c = v;
+        *out_size = 2;
+        return true;
+    } else if ((*buf & 0x80) == 0) {
+        int32_t v = 0;
+        v |= (buf[0] & 0x7F);
+        *c = v;
+        *out_size = 1;
+        return true;
+    }
+
+    return false;
+}
+
+// UTF-16 encoding, when U in U+010000 to U+10FFFF:
+//
+//  U' = yyyyyyyyyyxxxxxxxxxx  // U - 0x10000
+//  W1 = 110110yyyyyyyyyy      // 0xD800 + yyyyyyyyyy
+//  W2 = 110111xxxxxxxxxx      // 0xDC00 + xxxxxxxxxx
+
 bool bitstring_utf16_encode(avm_int_t c, uint8_t *buf, enum BitstringFlags bs_flags, size_t *out_size)
 {
     size_t sz = 0;
-    if (c < 0 || c > 0x10FFFF) {
+    if (is_invalid_codepoint(c)) {
         return false;
     }
     if (c < 0x10000) {
@@ -158,10 +236,66 @@ bool bitstring_utf16_encode(avm_int_t c, uint8_t *buf, enum BitstringFlags bs_fl
     return true;
 }
 
+bool bitstring_utf16_decode(const uint8_t *buf, size_t len, int32_t *c, size_t *out_size, enum BitstringFlags bs_flags)
+{
+    if (len == 0) {
+        return false;
+    } else if (bs_flags & LittleEndianIntegerMask) {
+        if (len >= 4 && ((buf[1] & 0xFC) == 0xD8) && ((buf[3] & 0xFC) == 0xDC)) {
+            int32_t v = 0;
+            v |= (buf[1] & 0x03) << 18;
+            v |= (buf[0] & 0xFF) << 10;
+            v |= (buf[3] & 0x03) << 8;
+            v |= (buf[2] & 0xFF);
+            v += 0x10000;
+            if (is_invalid_codepoint(v)) {
+                return false;
+            }
+            *c = v;
+            *out_size = 4;
+            return true;
+        } else if (len >= 2) {
+            int32_t v = 0;
+            v = READ_16LE_UNALIGNED(buf);
+            if (is_invalid_codepoint(v)) {
+                return false;
+            }
+            *c = v;
+            *out_size = 2;
+            return true;
+        }
+    } else {
+        if (len >= 4 && ((buf[0] & 0xFC) == 0xD8) && ((buf[2] & 0xFC) == 0xDC)) {
+            int32_t v = 0;
+            v |= (buf[0] & 0x03) << 18;
+            v |= (buf[1] & 0xFF) << 10;
+            v |= (buf[2] & 0x03) << 8;
+            v |= (buf[3] & 0xFF);
+            v += 0x10000;
+            if (is_invalid_codepoint(v)) {
+                return false;
+            }
+            *c = v;
+            *out_size = 4;
+            return true;
+        } else if (len >= 2) {
+            int32_t v = 0;
+            v = READ_16_UNALIGNED(buf);
+            if (is_invalid_codepoint(v)) {
+                return false;
+            }
+            *c = v;
+            *out_size = 2;
+            return true;
+        }
+    }
+    return false;
+}
+
 bool bitstring_utf32_encode(avm_int_t c, uint8_t *buf, enum BitstringFlags bs_flags)
 {
     UNUSED(bs_flags);
-    if (c < 0 || c > 0x10FFFF) {
+    if (is_invalid_codepoint(c)) {
         return false;
     }
     if (bs_flags & LittleEndianIntegerMask) {
@@ -177,3 +311,32 @@ bool bitstring_utf32_encode(avm_int_t c, uint8_t *buf, enum BitstringFlags bs_fl
     }
     return true;
 }
+
+bool bitstring_utf32_decode(const uint8_t *buf, size_t len, int32_t *c, enum BitstringFlags bs_flags)
+{
+    if (len < 4) {
+        return false;
+    } else if (bs_flags & LittleEndianIntegerMask) {
+        int32_t v = 0;
+        v |= (buf[3] & 0xFF) << 24;
+        v |= (buf[2] & 0xFF) << 16;
+        v |= (buf[1] & 0xFF) << 8;
+        v |=  buf[0] & 0xFF;
+        if (is_invalid_codepoint(v)) {
+            return false;
+        }
+        *c = v;
+        return true;
+    } else {
+        int32_t v = 0;
+        v |= (buf[0] & 0xFF) << 24;
+        v |= (buf[1] & 0xFF) << 16;
+        v |= (buf[2] & 0xFF) << 8;
+        v |=  buf[3] & 0xFF;
+        if (is_invalid_codepoint(v)) {
+            return false;
+        }
+        *c = v;
+        return true;
+    }
+}

src/libAtomVM/bitstring.h

@@ -313,6 +313,19 @@ static inline bool bitstring_insert_integer(term dst_bin, size_t offset, avm_int
  */
 bool bitstring_utf8_encode(avm_int_t c, uint8_t *buf, size_t *out_size);
 
+/**
+ * @brief Decode a character from UTF-8.
+ *
+ * @param c int value to decode to
+ * @param buf the buffer froom which to decode the sring to or NULL to only compute the
+ * size.
+ * @param len the length (in bytes) of the bytes in buf
+ * @param out_size the size in bytes, on output (if not NULL)
+ * @return \c true if decoding was successful, \c false if character starting at buf is not a valid
+ * unicode character
+ */
+bool bitstring_utf8_decode(const uint8_t *buf, size_t len, int32_t *c, size_t *out_size);
+
 /**
  * @brief Encode a character to UTF-16.
  *
@@ -326,6 +339,20 @@ bool bitstring_utf8_encode(avm_int_t c, uint8_t *buf, size_t *out_size);
  */
 bool bitstring_utf16_encode(avm_int_t c, uint8_t *buf, enum BitstringFlags bs_flags, size_t *out_size);
 
+/**
+ * @brief Decode a character from UTF-16.
+ *
+ * @param c int value to decode to
+ * @param buf the buffer froom which to decode the sring to or NULL to only compute the
+ * size.
+ * @param len the length (in bytes) of the bytes in buf
+ * @param bs_flags flags to decode the character (undefined/little/big/native)
+ * @param out_size the size in bytes, on output (if not NULL)
+ * @return \c true if decoding was successful, \c false if character starting at buf is not a valid
+ * unicode character
+ */
+bool bitstring_utf16_decode(const uint8_t *buf, size_t len, int32_t *c, size_t *out_size, enum BitstringFlags bs_flags);
+
 /**
  * @brief Encode a character to UTF-32.
  *
@@ -337,6 +364,19 @@ bool bitstring_utf16_encode(avm_int_t c, uint8_t *buf, enum BitstringFlags bs_fl
  */
 bool bitstring_utf32_encode(avm_int_t c, uint8_t *buf, enum BitstringFlags bs_flags);
 
+/**
+ * @brief Decode a character from UTF-32.
+ *
+ * @param c int value to decode to
+ * @param buf the buffer froom which to decode the sring to or NULL to only compute the
+ * size.
+ * @param len the length (in bytes) of the bytes in buf
+ * @param bs_flags flags to decode the character (undefined/little/big/native)
+ * @return \c true if decoding was successful, \c false if character starting at buf is not a valid
+ * unicode character
+ */
+bool bitstring_utf32_decode(const uint8_t *buf, size_t len, int32_t *c, enum BitstringFlags bs_flags);
+
 /**
  * @brief Compute the size of a character when UTF-8 encoded.
  *
@@ -379,6 +419,23 @@ static inline bool bitstring_insert_utf8(term dst_bin, size_t offset, avm_int_t
     return bitstring_utf8_encode(c, dst, out_size);
 }
 
+/**
+ * @brief Match a character in UTF-8 format
+ *
+ * @param src_bin binary to match against
+ * @param offset offset, in bits, to where to start to match the character
+ * @param c int to decode to
+ * @param out_size the size in bytes, on output
+ * @return \c true if encoding was successful, \c false if src_bin at offset is not a valid
+ * unicode character
+ */
+static inline bool bitstring_match_utf8(term src_bin, size_t offset, int32_t *c, size_t *out_size)
+{
+    size_t byte_offset = offset >> 3; // divide by 8
+    const uint8_t *src = (const uint8_t *) term_binary_data(src_bin) + byte_offset;
+    return bitstring_utf8_decode(src, term_binary_size(src_bin) - byte_offset, c, out_size);
+}
+
 /**
  * @brief Insert a character in UTF-&§ format
  *
@@ -397,6 +454,24 @@ static inline bool bitstring_insert_utf16(term dst_bin, size_t offset, avm_int_t
     return bitstring_utf16_encode(c, dst, bs_flags, out_size);
 }
 
+/**
+ * @brief Match a character in UTF-16 format
+ *
+ * @param src_bin binary to match against
+ * @param offset offset, in bits, to where to start to match the character
+ * @param c int to decode to
+ * @param bs_flags flags to decode the character (undefined/little/big/native)
+ * @param out_size the size in bytes, on output
+ * @return \c true if encoding was successful, \c false if src_bin at offset is not a valid
+ * unicode character
+ */
+static inline bool bitstring_match_utf16(term src_bin, size_t offset, int32_t *c, size_t *out_size, enum BitstringFlags bs_flags)
+{
+    size_t byte_offset = offset >> 3; // divide by 8
+    const uint8_t *src = (const uint8_t *) term_binary_data(src_bin) + byte_offset;
+    return bitstring_utf16_decode(src, term_binary_size(src_bin) - byte_offset, c, out_size, bs_flags);
+}
+
 /**
  * @brief Insert a character in UTF-32 format
  *
@@ -414,6 +489,23 @@ static inline bool bitstring_insert_utf32(term dst_bin, size_t offset, avm_int_t
     return bitstring_utf32_encode(c, dst, bs_flags);
 }
 
+/**
+ * @brief Match a character in UTF-32 format
+ *
+ * @param src_bin binary to match against
+ * @param offset offset, in bits, to where to start to match the character
+ * @param c int to decode to
+ * @param bs_flags flags to decode the character (undefined/little/big/native)
+ * @return \c true if encoding was successful, \c false if src_bin at offset is not a valid
+ * unicode character
+ */
+static inline bool bitstring_match_utf32(term src_bin, size_t offset, int32_t *c, enum BitstringFlags bs_flags)
+{
+    size_t byte_offset = offset >> 3; // divide by 8
+    const uint8_t *src = (const uint8_t *) term_binary_data(src_bin) + byte_offset;
+    return bitstring_utf32_decode(src, term_binary_size(src_bin) - byte_offset, c, bs_flags);
+}
+
 #ifdef __cplusplus
 }
 #endif

src/libAtomVM/opcodes.h

@@ -126,6 +126,12 @@
 #define OP_BS_PRIVATE_APPEND 135
 #define OP_TRIM 136
 #define OP_BS_INIT_BITS 137
+#define OP_BS_GET_UTF8 138
+#define OP_BS_SKIP_UTF8 139
+#define OP_BS_GET_UTF16 140
+#define OP_BS_SKIP_UTF16 141
+#define OP_BS_GET_UTF32 142
+#define OP_BS_SKIP_UTF32 143
 #define OP_BS_UTF8_SIZE 144
 #define OP_BS_PUT_UTF8 145
 #define OP_BS_UTF16_SIZE 146