Merge pull request atomvm#492 from fadushin/small_big_ext

bettio · bettio · commit 88d45118ee91 · 2023-03-24T00:24:12.000+01:00
Add support for SMALL_BIG_EXT term encoding

This PR adds support for the SMALL_BIG_EXT term encoding, supporting encoding
and decoding of all integer sizes currently supported by AtomVM.

These changes are made under both the "Apache 2.0" and the "GNU Lesser General
Public License 2.1 or later" license terms (dual license).

SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later
diff --git a/src/libAtomVM/externalterm.c b/src/libAtomVM/externalterm.c
@@ -40,6 +40,7 @@
 #define STRING_EXT 107
 #define LIST_EXT 108
 #define BINARY_EXT 109
+#define SMALL_BIG_EXT 110
 #define EXPORT_EXT 113
 #define MAP_EXT 116
 #define SMALL_ATOM_UTF8_EXT 119
@@ -48,13 +49,17 @@
 #define NEW_FLOAT_EXT_SIZE 9
 #define SMALL_INTEGER_EXT_SIZE 2
 #define INTEGER_EXT_SIZE 5
+#define SMALL_BIG_EXT_BASE_SIZE 3
 #define ATOM_EXT_BASE_SIZE 3
 #define STRING_EXT_BASE_SIZE 3
 #define LIST_EXT_BASE_SIZE 5
 #define BINARY_EXT_BASE_SIZE 5
 #define MAP_EXT_BASE_SIZE 5
 #define SMALL_ATOM_EXT_BASE_SIZE 2
 
+// Assuming two's-complement implementation of signed integers
+#define SIGNED_INT_TO_UNSIGNED(val, unsigned_type) ((val) < 0 ? ~((unsigned_type) (val)) + 1 : (val))
+
 // MAINTENANCE NOTE.  Range checking on the external term buffer is only performed in
 // the calculate_heap_usage function, which will fail with an invalid term if there is
 // insufficient space in the external term buffer (preventing reading off the end of the
@@ -202,6 +207,27 @@ static size_t compute_external_size(Context *ctx, term t)
     return serialize_term(ctx, NULL, t);
 }
 
+static uint8_t get_num_bytes(avm_uint64_t val)
+{
+    uint8_t num_bytes = 0;
+    while (val != 0) {
+        val = val >> 8;
+        ++num_bytes;
+    }
+    return num_bytes;
+}
+
+static void write_bytes(uint8_t *buf, avm_uint64_t val)
+{
+    uint8_t i = 0;
+    while (val != 0) {
+        uint8_t byte = val & 0xFF;
+        buf[i] = byte;
+        val = val >> 8;
+        ++i;
+    }
+}
+
 static int serialize_term(Context *ctx, uint8_t *buf, term t)
 {
     if (term_is_uint8(t)) {
@@ -211,13 +237,26 @@ static int serialize_term(Context *ctx, uint8_t *buf, term t)
         }
         return 2;
 
-    } else if (term_is_integer(t)) {
-        if (!IS_NULL_PTR(buf)) {
-            int32_t val = term_to_int32(t);
-            buf[0] = INTEGER_EXT;
-            WRITE_32_UNALIGNED(buf + 1, val);
+    } else if (term_is_any_integer(t)) {
+
+        avm_int64_t val = term_maybe_unbox_int64(t);
+        if (val >= INT32_MIN && val <= INT32_MAX) {
+            if (buf != NULL) {
+                buf[0] = INTEGER_EXT;
+                WRITE_32_UNALIGNED(buf + 1, (int32_t) val);
+            }
+            return INTEGER_EXT_SIZE;
+        } else {
+            avm_uint64_t unsigned_val = SIGNED_INT_TO_UNSIGNED(val, avm_uint64_t);
+            uint8_t num_bytes = get_num_bytes(unsigned_val);
+            if (buf != NULL) {
+                buf[0] = SMALL_BIG_EXT;
+                buf[1] = num_bytes;
+                buf[2] = val < 0 ? 0x01 : 0x00;
+                write_bytes(buf + 3, unsigned_val);
+            }
+            return SMALL_BIG_EXT_BASE_SIZE + num_bytes;
         }
-        return 5;
 
     } else if (term_is_float(t)) {
         if (!IS_NULL_PTR(buf)) {
@@ -341,6 +380,15 @@ static int serialize_term(Context *ctx, uint8_t *buf, term t)
     }
 }
 
+static avm_uint64_t read_bytes(const uint8_t *buf, uint8_t num_bytes)
+{
+    avm_uint64_t value = 0;
+    for (uint8_t i = 0; i < num_bytes; ++i) {
+        value |= (((avm_uint64_t) buf[i]) << (i * 8));
+    }
+    return value;
+}
+
 static term parse_external_terms(const uint8_t *external_term_buf, int *eterm_size, Context *ctx, bool copy)
 {
     switch (external_term_buf[0]) {
@@ -368,6 +416,23 @@ static term parse_external_terms(const uint8_t *external_term_buf, int *eterm_si
             return term_make_maybe_boxed_int64(ctx, value);
         }
 
+        case SMALL_BIG_EXT: {
+            uint8_t num_bytes = external_term_buf[1];
+            uint8_t sign = external_term_buf[2];
+            avm_uint64_t unsigned_value = read_bytes(external_term_buf + 3, num_bytes);
+            // NB due to call to calculate_heap_usage, there is no loss of precision:
+            // 1. 0 <= unsigned_value <= INT64_MAX if sign is 0
+            // 2. 0 <= unsigned_value <= INT64_MAX + 1 if sign is not 0
+            avm_int64_t value = 0;
+            if (sign != 0x00) {
+                value = -((avm_int64_t) unsigned_value);
+            } else {
+                value = (avm_int64_t) unsigned_value;
+            }
+            *eterm_size = SMALL_BIG_EXT_BASE_SIZE + num_bytes;
+            return term_make_maybe_boxed_int64(ctx, value);
+        }
+
         case ATOM_EXT: {
             uint16_t atom_len = READ_16_UNALIGNED(external_term_buf + 1);
 
@@ -549,6 +614,21 @@ static int calculate_heap_usage(const uint8_t *external_term_buf, size_t remaini
             return term_boxed_integer_size(value);
         }
 
+        case SMALL_BIG_EXT: {
+            uint8_t num_bytes = external_term_buf[1];
+            if (UNLIKELY(num_bytes > 8 || remaining < (SMALL_BIG_EXT_BASE_SIZE + num_bytes))) {
+                return INVALID_TERM_SIZE;
+            }
+            uint8_t sign = external_term_buf[2];
+            *eterm_size = SMALL_BIG_EXT_BASE_SIZE + num_bytes;
+            avm_uint64_t value = read_bytes(external_term_buf + 3, num_bytes);
+            // NB.  We currently support max 64-bit signed integers (assuming two's complement signed values in 63 bits)
+            if (UNLIKELY((sign == 0 && value > INT64_MAX) || (sign != 0 && value > (((avm_uint64_t) INT64_MAX) + 1)))) {
+                return INVALID_TERM_SIZE;
+            }
+            return term_boxed_integer_size(value);
+        }
+
         case ATOM_EXT: {
             if (UNLIKELY(remaining < ATOM_EXT_BASE_SIZE)) {
                 return INVALID_TERM_SIZE;
diff --git a/tests/erlang_tests/CMakeLists.txt b/tests/erlang_tests/CMakeLists.txt
@@ -439,6 +439,7 @@ compile_erlang(unlink_error)
 compile_erlang(trap_exit_flag)
 
 compile_erlang(test_stacktrace)
+compile_erlang(small_big_ext)
 
 add_custom_target(erlang_test_modules DEPENDS
     add.beam
@@ -853,4 +854,5 @@ add_custom_target(erlang_test_modules DEPENDS
     trap_exit_flag.beam
 
     test_stacktrace.beam
+    small_big_ext.beam
 )
diff --git a/tests/erlang_tests/small_big_ext.erl b/tests/erlang_tests/small_big_ext.erl
@@ -0,0 +1,115 @@
+%
+% This file is part of AtomVM.
+%
+% Copyright 2023 Fred Dushin <fred@dushin.net>
+%
+% Licensed under the Apache License, Version 2.0 (the "License");
+% you may not use this file except in compliance with the License.
+% You may obtain a copy of the License at
+%
+%    http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS,
+% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+% See the License for the specific language governing permissions and
+% limitations under the License.
+%
+% SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later
+%
+
+-module(small_big_ext).
+
+-export([start/0]).
+
+-define(INT64_MAX, 9223372036854775807).
+-define(INT64_MIN, -9223372036854775808).
+
+start() ->
+    true = test_reverse(pow(32), <<131, 110, 5, 0, 0, 0, 0, 0, 1>>),
+    true = test_reverse(pow(32) + 1, <<131, 110, 5, 0, 1, 0, 0, 0, 1>>),
+    true = test_reverse(pow(60) + 1, <<131, 110, 8, 0, 1, 0, 0, 0, 0, 0, 0, 16>>),
+    true = test_reverse(-1 * (pow(60) + 1), <<131, 110, 8, 1, 1, 0, 0, 0, 0, 0, 0, 16>>),
+    true = test_reverse(?INT64_MAX, <<131, 110, 8, 0, 255, 255, 255, 255, 255, 255, 255, 127>>),
+    true = test_reverse(?INT64_MIN, <<131, 110, 8, 1, 0, 0, 0, 0, 0, 0, 0, 128>>),
+
+    %% we can still decode really small encodings
+    1 = erlang:binary_to_term(<<131, 110, 1, 0, 1>>),
+    -1 = erlang:binary_to_term(<<131, 110, 1, 1, 1>>),
+    -1 = erlang:binary_to_term(<<131, 110, 1, 3, 1>>),
+    -1 = erlang:binary_to_term(<<131, 110, 1, -1, 1>>),
+
+    %% 0-length encodings are legal
+    0 = erlang:binary_to_term(<<131, 110, 0, 0>>),
+    0 = erlang:binary_to_term(<<131, 110, 0, 1>>),
+
+    %% edge cases around INT32 min/max boundaries
+    true = test_reverse(pow(31) - 1, <<131, 98, 127, 255, 255, 255>>),
+    true = test_reverse(pow(31), <<131, 110, 4, 0, 0, 0, 0, 128>>),
+    true = test_reverse(-pow(31), <<131, 98, 128, 0, 0, 0>>),
+    true = test_reverse(-pow(31) - 1, <<131, 110, 4, 1, 1, 0, 0, 128>>),
+
+    %% missing sign
+    ok = assert_badarg(
+        fun() ->
+            erlang:binary_to_term(<<131, 110, 0>>)
+        end
+    ),
+
+    %% we currently only support up to 64 bit (signed) integers
+    case erlang:system_info(machine) of
+        "BEAM" ->
+            test_reverse(
+                pow(63) + 1, <<131, 110, 8, 0, 1, 0, 0, 0, 0, 0, 0, 128>>
+            ),
+            test_reverse(
+                -(pow(63) + 2), <<131, 110, 8, 1, 2, 0, 0, 0, 0, 0, 0, 128>>
+            ),
+            test_reverse(
+                pow(128), <<131, 110, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1>>
+            );
+        _ ->
+            ok = assert_badarg(
+                fun() ->
+                    erlang:binary_to_term(
+                        <<131, 110, 8, 0, 1, 0, 0, 0, 0, 0, 0, 128>>
+                    )
+                end
+            ),
+            ok = assert_badarg(
+                fun() ->
+                    erlang:binary_to_term(
+                        <<131, 110, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1>>
+                    )
+                end
+            ),
+            ok = assert_badarg(
+                fun() ->
+                    erlang:binary_to_term(
+                        <<131, 110, 8, 1, 2, 0, 0, 0, 0, 0, 0, 128>>
+                    )
+                end
+            )
+    end,
+    0.
+
+test_reverse(T, Interop) ->
+    Bin = erlang:term_to_binary(T),
+    Bin = Interop,
+    {X, Used} = erlang:binary_to_term(Bin, [used]),
+    Used = erlang:byte_size(Bin),
+    X =:= T.
+
+assert_badarg(F) ->
+    try
+        R = F(),
+        {fail_no_ex, R}
+    catch
+        error:badarg -> ok
+    end.
+
+pow(0) ->
+    1;
+pow(X) ->
+    Y = pow(X - 1),
+    Y bsl 1.
diff --git a/tests/test.c b/tests/test.c
@@ -475,6 +475,7 @@ struct Test tests[] = {
     TEST_CASE_EXPECTED(unlink_error, 1),
     TEST_CASE_EXPECTED(trap_exit_flag, 1),
     TEST_CASE_COND(test_stacktrace, 0, SKIP_STACKTRACES),
+    TEST_CASE(small_big_ext),
 
     // TEST CRASHES HERE: TEST_CASE(memlimit),
 
